tests(refactor): Conversion to parallel tests and use revised helpers

- Introduced `CONTAINER_NAME` and `TEST_NAME_PREFIX` as new vars for better managing test consistency (DRY).
- `CONTAINER_NAME` replaces any repeated container name with the variable. The value will differ slightly as the prior prefix (`mail_`) has been changed to `dms-test-`.
- `TEST_NAME_PREFIX` provides a prefix value for each `@test` description string.

---

chore: Add a reference template for tests
This commit is contained in:
Brennan Kinney 2022-11-26 10:59:14 +13:00
parent 32cc9d30e5
commit 75ee0c1145
7 changed files with 158 additions and 161 deletions

View file

@ -3,29 +3,22 @@ load "${REPOSITORY_ROOT}/test/helper/common"
TEST_NAME_PREFIX='ClamAV:'
CONTAINER_NAME='dms-test-clamav'
RUN_COMMAND=('run' 'docker' 'exec' "${CONTAINER_NAME}")
function setup_file() {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container . "${CONTAINER_NAME}")
init_with_defaults
docker run --rm --detach --tty \
--name "${CONTAINER_NAME}" \
--hostname mail.my-domain.com \
--volume "${PRIVATE_CONFIG}:/tmp/docker-mailserver" \
--volume "${PWD}/test/test-files:/tmp/docker-mailserver-test:ro" \
--env ENABLE_AMAVIS=1 \
--env AMAVIS_LOGLEVEL=2 \
--env ENABLE_CLAMAV=1 \
--env ENABLE_UPDATE_CHECK=0 \
--env ENABLE_SPAMASSASSIN=0 \
--env ENABLE_FAIL2BAN=0 \
--env PERMIT_DOCKER=host \
--env CLAMAV_MESSAGE_SIZE_LIMIT=30M \
--env LOG_LEVEL=debug \
"${IMAGE_NAME}"
# Comment for maintainers about `PERMIT_DOCKER=host`:
# https://github.com/docker-mailserver/docker-mailserver/pull/2815/files#r991087509
local CUSTOM_SETUP_ARGUMENTS=(
--env ENABLE_CLAMAV=1
--env ENABLE_AMAVIS=1
--env PERMIT_DOCKER=host
--env AMAVIS_LOGLEVEL=2
--env CLAMAV_MESSAGE_SIZE_LIMIT=30M
--env LOG_LEVEL=trace
)
wait_for_finished_setup_in_container "${CONTAINER_NAME}"
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
# wait for ClamAV to be fully setup or we will get errors on the log
repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl
@ -33,48 +26,46 @@ function setup_file() {
wait_for_service "${CONTAINER_NAME}" postfix
wait_for_smtp_port_in_container "${CONTAINER_NAME}"
"${RUN_COMMAND[@]}" bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt"
_run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt"
assert_success
wait_for_empty_mail_queue_in_container "${CONTAINER_NAME}"
}
function teardown_file() {
docker rm -f "${CONTAINER_NAME}"
}
function teardown_file() { _default_teardown ; }
@test "${TEST_NAME_PREFIX} process clamd is running" {
"${RUN_COMMAND[@]}" bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
_run_in_container bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
assert_success
}
@test "${TEST_NAME_PREFIX} log files exist at /var/log/mail directory" {
"${RUN_COMMAND[@]}" bash -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log'| wc -l"
_run_in_container bash -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l"
assert_success
assert_output 3
}
@test "${TEST_NAME_PREFIX} should be identified by Amavis" {
"${RUN_COMMAND[@]}" grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
_run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
assert_success
}
@test "${TEST_NAME_PREFIX} freshclam cron is enabled" {
"${RUN_COMMAND[@]}" bash -c "grep '/usr/bin/freshclam' -r /etc/cron.d"
_run_in_container bash -c "grep '/usr/bin/freshclam' -r /etc/cron.d"
assert_success
}
@test "${TEST_NAME_PREFIX} env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" {
"${RUN_COMMAND[@]}" grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf
_run_in_container grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf
assert_success
}
@test "${TEST_NAME_PREFIX} rejects virus" {
"${RUN_COMMAND[@]}" bash -c "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep '<virus@external.tld> -> <user1@localhost.localdomain>'"
_run_in_container bash -c "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep '<virus@external.tld> -> <user1@localhost.localdomain>'"
assert_success
}
@test "${TEST_NAME_PREFIX} process clamd restarts when killed" {
"${RUN_COMMAND[@]}" bash -c "pkill clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
_run_in_container bash -c "pkill clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
assert_success
}

View file

@ -1,29 +1,23 @@
load "${REPOSITORY_ROOT}/test/helper/setup"
load "${REPOSITORY_ROOT}/test/helper/common"
function setup() {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container relay-hosts)
export TEST_NAME_PREFIX='default relay host:'
export CONTAINER_NAME='dms-test-default_relay_host'
docker run -d --name mail_with_default_relay \
-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
-v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \
-e DEFAULT_RELAY_HOST=default.relay.host.invalid:25 \
-e PERMIT_DOCKER=host \
-h mail.my-domain.com -t "${NAME}"
function setup_file() {
init_with_defaults
wait_for_finished_setup_in_container mail_with_default_relay
local CUSTOM_SETUP_ARGUMENTS=(
--env DEFAULT_RELAY_HOST=default.relay.host.invalid:25 \
--env PERMIT_DOCKER=host \
)
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
}
function teardown() {
docker rm -f mail_with_default_relay
}
function teardown_file() { _default_teardown ; }
#
# default relay host
#
@test "checking default relay host: default relay host is added to main.cf" {
run docker exec mail_with_default_relay /bin/sh -c 'grep -e "^relayhost =" /etc/postfix/main.cf'
@test "${TEST_NAME_PREFIX} default relay host is added to main.cf" {
_run_in_container bash -c 'grep -e "^relayhost =" /etc/postfix/main.cf'
assert_output 'relayhost = default.relay.host.invalid:25'
}

View file

@ -1,48 +1,36 @@
load "${REPOSITORY_ROOT}/test/helper/setup"
load "${REPOSITORY_ROOT}/test/helper/common"
# Can run tests in parallel?: No
# Shared static container name: TEST_NAME
TEST_NAME_PREFIX='spam (Amavis):'
CONTAINER_NAME='dms-test-spam_bounced'
function setup_file() {
init_with_defaults
local CUSTOM_SETUP_ARGUMENTS=(
--env ENABLE_AMAVIS=1
--env ENABLE_SPAMASSASSIN=1
--env PERMIT_DOCKER=container
--env SPAMASSASSIN_SPAM_TO_INBOX=0
)
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
wait_for_smtp_port_in_container_to_respond "${CONTAINER_NAME}"
}
function teardown_file() { _default_teardown ; }
# Test case
# ---------
# When SPAMASSASSIN_SPAM_TO_INBOX=0, spam messages must be bounced (rejected).
# SPAMASSASSIN_SPAM_TO_INBOX=1 is covered in `mail_spam_junk_folder.bats`.
# Original test PR: https://github.com/docker-mailserver/docker-mailserver/pull/1485
function teardown() {
docker rm -f "${TEST_NAME}"
}
function setup_file() {
init_with_defaults
}
# Not used
# function teardown_file() {
# }
@test "checking amavis: spam message is bounced (rejected)" {
# shellcheck disable=SC2034
local TEST_DOCKER_ARGS=(
--env ENABLE_SPAMASSASSIN=1
--env PERMIT_DOCKER=container
--env SPAMASSASSIN_SPAM_TO_INBOX=0
)
common_container_setup 'TEST_DOCKER_ARGS'
_should_bounce_spam
}
function _should_bounce_spam() {
wait_for_smtp_port_in_container_to_respond "${TEST_NAME}"
@test "${TEST_NAME_PREFIX} spam message is bounced (rejected)" {
# send a spam message
run docker exec "${TEST_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt"
_run_in_container /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt"
assert_success
# message will be added to a queue with varying delay until amavis receives it
run repeat_until_success_or_timeout 60 sh -c "docker logs ${TEST_NAME} | grep 'Blocked SPAM {NoBounceInbound,Quarantined}'"
run repeat_until_success_or_timeout 60 sh -c "docker logs ${CONTAINER_NAME} | grep 'Blocked SPAM {NoBounceInbound,Quarantined}'"
assert_success
}

View file

@ -0,0 +1,39 @@
# ? load the BATS helper
load "${REPOSITORY_ROOT}/test/helper/setup"
load "${REPOSITORY_ROOT}/test/helper/common"
# ? global variable initialization
# ? to identify the test easily
TEST_NAME_PREFIX='template:'
# ? must be unique
CONTAINER_NAME='dms-test-template'
# ? test setup
function setup_file() {
# ? optional setup before container is started
# ? initialize the test helpers
init_with_defaults
# ? add custom arguments supplied to `docker run` here
local CUSTOM_SETUP_ARGUMENTS=(
--env LOG_LEVEL=trace
)
# ? use a helper to correctly setup the container
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
# ? optional setup after the container is started
}
# ? test finalization
function teardown_file() { _default_teardown ; }
# ? actual unit tests
@test "${TEST_NAME_PREFIX} default check" {
_run_in_container bash -c "true"
assert_success
}

View file

@ -1,55 +1,46 @@
load "${REPOSITORY_ROOT}/test/helper/setup"
load "${REPOSITORY_ROOT}/test/helper/common"
function setup_file() {
local PRIVATE_CONFIG
export ALL IPV4 IPV6
TEST_NAME_PREFIX='Dovecot protocols:'
PRIVATE_CONFIG=$(duplicate_config_for_container . "${IPV4}")
ALL="mail_dovecot_all_protocols"
IPV4="mail_dovecot_ipv4"
IPV6="mail_dovecot_ipv6"
@test "${TEST_NAME_PREFIX} dual-stack IP configuration" {
local CONTAINER_NAME='dms-test-dovecot_protocols_all'
local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=)
docker run --rm -d --name "${ALL}" \
-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
-e DOVECOT_INET_PROTOCOLS= \
-h mail.my-domain.com \
-t "${NAME}"
init_with_defaults
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
docker run --rm -d --name "${IPV4}" \
-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
-e DOVECOT_INET_PROTOCOLS=ipv4 \
-h mail.my-domain.com \
-t "${NAME}"
docker run --rm -d --name "${IPV6}" \
-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
-e DOVECOT_INET_PROTOCOLS=ipv6 \
-h mail.my-domain.com \
-t "${NAME}"
}
@test 'checking dovecot IP configuration' {
wait_for_finished_setup_in_container "${ALL}"
run docker exec "${ALL}" grep '^#listen = \*, ::' /etc/dovecot/dovecot.conf
_run_in_container grep '^#listen = \*, ::' /etc/dovecot/dovecot.conf
assert_success
assert_output '#listen = *, ::'
docker rm -f "${CONTAINER_NAME}"
}
@test 'checking dovecot IPv4 configuration' {
wait_for_finished_setup_in_container "${IPV4}"
run docker exec "${IPV4}" grep '^listen = \*$' /etc/dovecot/dovecot.conf
@test "${TEST_NAME_PREFIX} IPv4 configuration" {
local CONTAINER_NAME='dms-test-dovecot_protocols_ipv4'
local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=ipv4)
init_with_defaults
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
_run_in_container grep '^listen = \*$' /etc/dovecot/dovecot.conf
assert_success
assert_output 'listen = *'
docker rm -f "${CONTAINER_NAME}"
}
@test 'checking dovecot IPv6 configuration' {
wait_for_finished_setup_in_container "${IPV6}"
run docker exec "${IPV6}" grep '^listen = \[::\]$' /etc/dovecot/dovecot.conf
@test "${TEST_NAME_PREFIX} IPv6 configuration" {
local CONTAINER_NAME='dms-test-dovecot_protocols_ipv6'
local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=ipv6)
init_with_defaults
common_container_setup 'CUSTOM_SETUP_ARGUMENTS'
_run_in_container grep '^listen = \[::\]$' /etc/dovecot/dovecot.conf
assert_success
assert_output 'listen = [::]'
}
function teardown_file {
docker rm -f "${ALL}" "${IPV4}" "${IPV6}"
docker rm -f "${CONTAINER_NAME}"
}

View file

@ -1,31 +1,23 @@
load "${REPOSITORY_ROOT}/test/helper/setup"
load "${REPOSITORY_ROOT}/test/helper/common"
TEST_NAME_PREFIX='helper functions inside container:'
CONTAINER_NAME='dms-test-helper_functions'
function setup_file() {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container .)
docker run -d --name mail_helper_functions \
--cap-add=NET_ADMIN \
-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
-v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \
-e ENABLE_FETCHMAIL=1 \
-h mail.my-domain.com -t "${NAME}"
wait_for_finished_setup_in_container mail_helper_functions
init_with_defaults
common_container_setup
}
function teardown_file() {
docker rm -f mail_helper_functions
}
@test "check helper functions (network.sh): _sanitize_ipv4_to_subnet_cidr" {
run docker exec mail_helper_functions bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 255.255.255.255/0"
assert_output "0.0.0.0/0"
run docker exec mail_helper_functions bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/20"
assert_output "192.168.240.0/20"
run docker exec mail_helper_functions bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/32"
assert_output "192.168.255.14/32"
function teardown_file() { _default_teardown ; }
@test "${TEST_NAME_PREFIX} _sanitize_ipv4_to_subnet_cidr" {
_run_in_container bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 255.255.255.255/0"
assert_output "0.0.0.0/0"
_run_in_container bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/20"
assert_output "192.168.240.0/20"
_run_in_container bash -c "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/32"
assert_output "192.168.255.14/32"
}

View file

@ -1,12 +1,14 @@
load "${REPOSITORY_ROOT}/test/test_helper/common"
@test "repeat_until_success_or_timeout returns instantly on success" {
TEST_NAME_PREFIX='test helper functions:'
@test "${TEST_NAME_PREFIX} repeat_until_success_or_timeout returns instantly on success" {
SECONDS=0
repeat_until_success_or_timeout 1 true
[[ ${SECONDS} -le 1 ]]
}
@test "repeat_until_success_or_timeout waits for timeout on persistent failure" {
@test "${TEST_NAME_PREFIX} repeat_until_success_or_timeout waits for timeout on persistent failure" {
SECONDS=0
run repeat_until_success_or_timeout 2 false
[[ ${SECONDS} -ge 2 ]]
@ -14,7 +16,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_output --partial "Timed out on command"
}
@test "repeat_until_success_or_timeout aborts immediately on fatal failure" {
@test "${TEST_NAME_PREFIX} repeat_until_success_or_timeout aborts immediately on fatal failure" {
SECONDS=0
run repeat_until_success_or_timeout --fatal-test false 2 false
[[ ${SECONDS} -le 1 ]]
@ -22,7 +24,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_output --partial "early aborting"
}
@test "repeat_until_success_or_timeout expects integer timeout" {
@test "${TEST_NAME_PREFIX} repeat_until_success_or_timeout expects integer timeout" {
run repeat_until_success_or_timeout 1 true
assert_success
@ -33,27 +35,27 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_failure
}
@test "run_until_success_or_timeout returns instantly on success" {
@test "${TEST_NAME_PREFIX} run_until_success_or_timeout returns instantly on success" {
SECONDS=0
run_until_success_or_timeout 2 true
[[ ${SECONDS} -le 1 ]]
assert_success
}
@test "run_until_success_or_timeout waits for timeout on persistent failure" {
@test "${TEST_NAME_PREFIX} run_until_success_or_timeout waits for timeout on persistent failure" {
SECONDS=0
! run_until_success_or_timeout 2 false
[[ ${SECONDS} -ge 2 ]]
assert_failure
}
@test "repeat_in_container_until_success_or_timeout fails immediately for non-running container" {
@test "${TEST_NAME_PREFIX} repeat_in_container_until_success_or_timeout fails immediately for non-running container" {
SECONDS=0
! repeat_in_container_until_success_or_timeout 10 name-of-non-existing-container true
[[ ${SECONDS} -le 1 ]]
}
@test "repeat_in_container_until_success_or_timeout run command in container" {
@test "${TEST_NAME_PREFIX} repeat_in_container_until_success_or_timeout run command in container" {
local CONTAINER_NAME
CONTAINER_NAME=$(docker run --rm -d alpine sleep 100)
SECONDS=0
@ -63,7 +65,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_output "${CONTAINER_NAME}"
}
@test "container_is_running" {
@test "${TEST_NAME_PREFIX} container_is_running" {
local CONTAINER_NAME
CONTAINER_NAME=$(docker run --rm -d alpine sleep 100)
container_is_running "${CONTAINER_NAME}"
@ -71,7 +73,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
! container_is_running "${CONTAINER_NAME}"
}
@test "wait_for_smtp_port_in_container aborts wait after timeout" {
@test "${TEST_NAME_PREFIX} wait_for_smtp_port_in_container aborts wait after timeout" {
local CONTAINER_NAME
CONTAINER_NAME=$(docker run --rm -d alpine sleep 100)
SECONDS=0
@ -82,7 +84,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
}
# NOTE: Test requires external network access available
@test "wait_for_smtp_port_in_container returns immediately when port found" {
@test "${TEST_NAME_PREFIX} wait_for_smtp_port_in_container returns immediately when port found" {
local CONTAINER_NAME
CONTAINER_NAME=$(docker run --rm -d alpine sh -c "sleep 10")
@ -95,7 +97,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_success
}
@test "wait_for_finished_setup_in_container" {
@test "${TEST_NAME_PREFIX} wait_for_finished_setup_in_container" {
# variable not local to make visible to teardown
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container .)
@ -117,7 +119,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
[[ ${SECONDS} -gt 0 ]]
}
@test "duplicate_config_for_container" {
@test "${TEST_NAME_PREFIX} duplicate_config_for_container" {
local path
path=$(duplicate_config_for_container duplicate_config_test)
@ -128,7 +130,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_failure
}
@test "container_has_service_running/wait_for_service" {
@test "${TEST_NAME_PREFIX} container_has_service_running/wait_for_service" {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container .)
@ -156,7 +158,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
assert_failure
}
@test "wait_for_changes_to_be_detected_in_container fails when timeout is reached" {
@test "${TEST_NAME_PREFIX} wait_for_changes_to_be_detected_in_container fails when timeout is reached" {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container .)
@ -182,7 +184,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
! TEST_TIMEOUT_IN_SECONDS=0 wait_for_changes_to_be_detected_in_container "${CONTAINER_NAME}"
}
@test "wait_for_changes_to_be_detected_in_container succeeds within timeout" {
@test "${TEST_NAME_PREFIX} wait_for_changes_to_be_detected_in_container succeeds within timeout" {
local PRIVATE_CONFIG
PRIVATE_CONFIG=$(duplicate_config_for_container .)
@ -208,7 +210,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
}
# TODO investigate why this test fails
@test "wait_for_empty_mail_queue_in_container fails when timeout reached" {
@test "${TEST_NAME_PREFIX} wait_for_empty_mail_queue_in_container fails when timeout reached" {
skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2177'
local PRIVATE_CONFIG
@ -240,7 +242,7 @@ load "${REPOSITORY_ROOT}/test/test_helper/common"
}
# TODO investigate why this test fails
@test "wait_for_empty_mail_queue_in_container succeeds within timeout" {
@test "${TEST_NAME_PREFIX} wait_for_empty_mail_queue_in_container succeeds within timeout" {
skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2177'
local PRIVATE_CONFIG