From a81de22819a5bd9820cd0bc265d1fce709bbc9e2 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 24 Dec 2022 21:15:23 +1300 Subject: [PATCH 1/5] tests(chore): Move some serial tests into parallel sets Additionally with the `tls.bash` helper for the `letsencrypt` tests. --- test/{test_helper => helper}/tls.bash | 0 test/tests/parallel/set1/{ => spam_virus}/clamav.bats | 0 .../set1/spam_virus/disabled_clamav_spamassassin.bats} | 0 .../mail_dnsbl.bats => parallel/set1/spam_virus/dnsbl.bats} | 0 .../mail_fail2ban.bats => parallel/set1/spam_virus/fail2ban.bats} | 0 .../set1/spam_virus/postgrey_enabled.bats} | 0 .../set1/spam_virus/postscreen.bats} | 0 test/tests/parallel/{set2 => set1/spam_virus}/spam_bounced.bats | 0 .../set1/spam_virus/spam_junk_folder.bats} | 0 .../set1/spam_virus/undef_spam_subject.bats} | 0 .../set2/tls/tls_cipherlists.bats} | 0 .../set2/tls/tls_letsencrypt.bats} | 0 .../mail_ssl_manual.bats => parallel/set2/tls/tls_manual.bats} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename test/{test_helper => helper}/tls.bash (100%) rename test/tests/parallel/set1/{ => spam_virus}/clamav.bats (100%) rename test/tests/{serial/mail_disabled_clamav_spamassassin.bats => parallel/set1/spam_virus/disabled_clamav_spamassassin.bats} (100%) rename test/tests/{serial/mail_dnsbl.bats => parallel/set1/spam_virus/dnsbl.bats} (100%) rename test/tests/{serial/mail_fail2ban.bats => parallel/set1/spam_virus/fail2ban.bats} (100%) rename test/tests/{serial/mail_with_postgrey.bats => parallel/set1/spam_virus/postgrey_enabled.bats} (100%) rename test/tests/{serial/mail_postscreen.bats => parallel/set1/spam_virus/postscreen.bats} (100%) rename test/tests/parallel/{set2 => set1/spam_virus}/spam_bounced.bats (100%) rename test/tests/{serial/mail_spam_junk_folder.bats => parallel/set1/spam_virus/spam_junk_folder.bats} (100%) rename test/tests/{serial/mail_undef_spam_subject.bats => parallel/set1/spam_virus/undef_spam_subject.bats} (100%) rename test/tests/{serial/security_tls_cipherlists.bats => parallel/set2/tls/tls_cipherlists.bats} (100%) rename test/tests/{serial/mail_ssl_letsencrypt.bats => parallel/set2/tls/tls_letsencrypt.bats} (100%) rename test/tests/{serial/mail_ssl_manual.bats => parallel/set2/tls/tls_manual.bats} (100%) diff --git a/test/test_helper/tls.bash b/test/helper/tls.bash similarity index 100% rename from test/test_helper/tls.bash rename to test/helper/tls.bash diff --git a/test/tests/parallel/set1/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats similarity index 100% rename from test/tests/parallel/set1/clamav.bats rename to test/tests/parallel/set1/spam_virus/clamav.bats diff --git a/test/tests/serial/mail_disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats similarity index 100% rename from test/tests/serial/mail_disabled_clamav_spamassassin.bats rename to test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats diff --git a/test/tests/serial/mail_dnsbl.bats b/test/tests/parallel/set1/spam_virus/dnsbl.bats similarity index 100% rename from test/tests/serial/mail_dnsbl.bats rename to test/tests/parallel/set1/spam_virus/dnsbl.bats diff --git a/test/tests/serial/mail_fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats similarity index 100% rename from test/tests/serial/mail_fail2ban.bats rename to test/tests/parallel/set1/spam_virus/fail2ban.bats diff --git a/test/tests/serial/mail_with_postgrey.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats similarity index 100% rename from test/tests/serial/mail_with_postgrey.bats rename to test/tests/parallel/set1/spam_virus/postgrey_enabled.bats diff --git a/test/tests/serial/mail_postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats similarity index 100% rename from test/tests/serial/mail_postscreen.bats rename to test/tests/parallel/set1/spam_virus/postscreen.bats diff --git a/test/tests/parallel/set2/spam_bounced.bats b/test/tests/parallel/set1/spam_virus/spam_bounced.bats similarity index 100% rename from test/tests/parallel/set2/spam_bounced.bats rename to test/tests/parallel/set1/spam_virus/spam_bounced.bats diff --git a/test/tests/serial/mail_spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats similarity index 100% rename from test/tests/serial/mail_spam_junk_folder.bats rename to test/tests/parallel/set1/spam_virus/spam_junk_folder.bats diff --git a/test/tests/serial/mail_undef_spam_subject.bats b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats similarity index 100% rename from test/tests/serial/mail_undef_spam_subject.bats rename to test/tests/parallel/set1/spam_virus/undef_spam_subject.bats diff --git a/test/tests/serial/security_tls_cipherlists.bats b/test/tests/parallel/set2/tls/tls_cipherlists.bats similarity index 100% rename from test/tests/serial/security_tls_cipherlists.bats rename to test/tests/parallel/set2/tls/tls_cipherlists.bats diff --git a/test/tests/serial/mail_ssl_letsencrypt.bats b/test/tests/parallel/set2/tls/tls_letsencrypt.bats similarity index 100% rename from test/tests/serial/mail_ssl_letsencrypt.bats rename to test/tests/parallel/set2/tls/tls_letsencrypt.bats diff --git a/test/tests/serial/mail_ssl_manual.bats b/test/tests/parallel/set2/tls/tls_manual.bats similarity index 100% rename from test/tests/serial/mail_ssl_manual.bats rename to test/tests/parallel/set2/tls/tls_manual.bats From 306592fcad61c1d450e82efa99af24899d8657ef Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 3 Jan 2023 19:00:13 +1300 Subject: [PATCH 2/5] tests: Adjusted files not directly related to tests `tls.bash` helper was adapted to the new helper scripts location. The `setup.bash` helper saw a bugfix (expanding the array properly) and updates the container default config to configure for IPv4 explicitly. The IPv4 default was added after recent Docker pushes and I saw weird IPv6 related errors in the logs.. now we're sure IPv4 is the default during tests. Added functionality to check if a process is running: - This change adds a helper function to check whether a program is running inside a container or not. - This added the need for a function like `_run_in_container` but allowing for providing an explicit container name. - Future PRs can use this helper function now to check whether a process is running or not. This was done for the tests of Fail2Ban, but can be used for other tests in the future as well. --- chore: Restructured BATS flags in `Makefile` The `Makefile` has seen a bit of a restructuring when it comes to flags: 1. The `MAKEFLAGS` variables is used by `make`, and allows for adding additional flags that can be used within in recursive calls (via `$(MAKE)`) too, thus DRY approach. 2. The flags for calling BATS were adjusted. `--no-parallelize-within-files` has been added as well to ensure tests _inside_ a single file are run sequentially. `dms-test` prefix matching changed to expect a `_` suffix as a delimiter. --- docs: Add a note regarding output from running tests in parallel --- Makefile | 19 ++++++++++--------- docs/content/contributing/general.md | 14 ++++++++++++-- test/helper/common.bash | 19 +++++++++++++++++++ test/helper/setup.bash | 7 +++++-- test/helper/tls.bash | 15 ++++----------- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1577ab7c..2367954e 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ SHELL := /bin/bash .SHELLFLAGS += -e -u -o pipefail -PARALLEL_JOBS ?= 2 export REPOSITORY_ROOT := $(CURDIR) export IMAGE_NAME ?= mailserver-testing:ci export NAME ?= $(IMAGE_NAME) +MAKEFLAGS += --no-print-directory +BATS_FLAGS ?= --timing +BATS_PARALLEL_JOBS ?= 2 +BATS_FLAGS_PARALLEL ?= $(BATS_FLAGS) --no-parallelize-within-files --jobs $(BATS_PARALLEL_JOBS) + .PHONY: ALWAYS_RUN # ----------------------------------------------- @@ -33,7 +37,7 @@ backup: clean: # remove test containers and restore test/config directory -@ [[ -d testconfig.bak ]] && { sudo rm -rf test/config ; mv testconfig.bak test/config ; } || : - -@ for CONTAINER in $$(docker ps -a --filter name='^dms-test-.*|^mail_.*|^hadolint$$|^eclint$$|^shellcheck$$' | sed 1d | cut -f 1-1 -d ' '); do docker rm -f $${CONTAINER}; done + -@ for CONTAINER in $$(docker ps -a --filter name='^dms-test_.*|^mail_.*|^hadolint$$|^eclint$$|^shellcheck$$' | sed 1d | cut -f 1-1 -d ' '); do docker rm -f $${CONTAINER}; done -@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && sudo rm -rf $${LINE}; done < .gitignore # ----------------------------------------------- @@ -43,19 +47,16 @@ clean: tests: ALWAYS_RUN # See https://github.com/docker-mailserver/docker-mailserver/pull/2857#issuecomment-1312724303 # on why `generate-accounts` is run before each set (TODO/FIXME) - @ $(MAKE) generate-accounts tests/serial - @ $(MAKE) generate-accounts tests/parallel/set1 - @ $(MAKE) generate-accounts tests/parallel/set2 - @ $(MAKE) generate-accounts tests/parallel/set3 + @ for DIR in tests/{serial,parallel/set{1,2,3}} ; do $(MAKE) generate-accounts "$${DIR}" ; done tests/serial: ALWAYS_RUN - @ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs 1 test/$@/**.bats + @ shopt -s globstar ; ./test/bats/bin/bats $(BATS_FLAGS) test/$@/*.bats tests/parallel/set%: ALWAYS_RUN - @ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs $(PARALLEL_JOBS) test/$@/**.bats + @ shopt -s globstar ; ./test/bats/bin/bats $(BATS_FLAGS_PARALLEL) test/$@/**/*.bats test/%: ALWAYS_RUN - @ shopt -s globstar nullglob ; ./test/bats/bin/bats --timing test/tests/**/{$*,}.bats + @ shopt -s globstar nullglob ; ./test/bats/bin/bats $(BATS_FLAGS) test/tests/**/{$*,}.bats # ----------------------------------------------- # --- Lints ------------------------------------- diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md index 7270fcb8..d6bdcb94 100644 --- a/docs/content/contributing/general.md +++ b/docs/content/contributing/general.md @@ -24,11 +24,21 @@ To run the test suite, you will need to We do not support running linting, tests, etc on macOS at this time. Please use a linux VM. +??? tip "Setting the Degree of Parallelization for Tests" + + If your machine is capable, you can increase the amount of tests that are run simultaneously by prepending the `make clean all` command with `BATS_PARALLEL_JOBS=X` (i.e. `BATS_PARALLEL_JOBS=X make clean all`). This wil speed up the test procedure. You can also run all tests in serial by setting `BATS_PARALLEL_JOBS=1` this way. + + The default value of `BATS_PARALLEL_JOBS` is 2. Increasing it to `3` requires 6 threads and 6GB of main memory; increasing it to `4` requires 8 threads and at least 8GB of main memory. + +!!! warning "Test Output when Running in Parallel" + + When running test in parallel, BATS will run more than one test at any given time. This can result in output not being exactly what you'd expect, i.e. there could be _more_ or _less_ than you'd think. Those writing tests need to take care of this. Always test with `make clean generate-accounts tests/parallel/setX`. + ??? tip "Running a Specific Test" - To run a specific test, use `make build generate-accounts test/`, where `` is the file name of the test (_for more precision use a relative path: `test/test/`_) excluding the `.bats` suffix. + To run a specific test, use `make build generate-accounts test/`, where `` is the file name of the test (_for more precision use a relative path: `test/test/`_) **excluding** the `.bats` suffix. - To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`). + Example: To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`). [Install Docker]: https://docs.docker.com/get-docker/ diff --git a/test/helper/common.bash b/test/helper/common.bash index 6fc553f2..1f38b132 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -9,16 +9,35 @@ __load_bats_helper # ------------------------------------------------------------------- +# like _run_in_container_explicit but infers ${1} by using the ENV CONTAINER_NAME function _run_in_container() { run docker exec "${CONTAINER_NAME}" "${@}" } +# @param ${1} container name [REQUIRED] +# @param ... command to execute +function _run_in_container_explicit() { + local CONTAINER_NAME=${1:?Container name must be given when using explicit} + shift 1 + run docker exec "${CONTAINER_NAME}" "${@}" +} + function _default_teardown() { docker rm -f "${CONTAINER_NAME}" } # ------------------------------------------------------------------- +# @param ${1} program name [REQUIRED] +# @param ${2} container name [IF UNSET: ${CONTAINER_NAME}] +function _check_if_process_is_running() { + local PROGRAM_NAME=${1:?Program name must be provided explicitly} + local CONTAINER_NAME=${2:-${CONTAINER_NAME}} + _run_in_container_explicit "${CONTAINER_NAME}" pgrep "${PROGRAM_NAME}" +} + +# ------------------------------------------------------------------- + # @param ${1} timeout # @param --fatal-test additional test whose failure aborts immediately # @param ... test to run diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 522cfc84..4bab5925 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -17,14 +17,15 @@ function __initialize_variables() { 'CONTAINER_NAME' ) - for VARIABLE in "${REQUIRED_VARIABLES_FOR_TESTS}" + for VARIABLE in "${REQUIRED_VARIABLES_FOR_TESTS[@]}" do __check_if_set "${VARIABLE}" done + export SETUP_FILE_MARKER TEST_TIMEOUT_IN_SECONDS NUMBER_OF_LOG_LINES + SETUP_FILE_MARKER="${BATS_TMPDIR:?}/$(basename "${BATS_TEST_FILENAME:?}").setup_file" TEST_TIMEOUT_IN_SECONDS=${TEST_TIMEOUT_IN_SECONDS:-120} NUMBER_OF_LOG_LINES=${NUMBER_OF_LOG_LINES:-10} - SETUP_FILE_MARKER="${BATS_TMPDIR:?}/$(basename "${BATS_TEST_FILENAME:?}").setup_file" } # ------------------------------------------------------------------- @@ -118,6 +119,8 @@ function common_container_create() { --env ENABLE_UPDATE_CHECK=0 \ --env ENABLE_SPAMASSASSIN=0 \ --env ENABLE_FAIL2BAN=0 \ + --env POSTFIX_INET_PROTOCOLS=ipv4 \ + --env DOVECOT_INET_PROTOCOLS=ipv4 \ --env LOG_LEVEL=debug \ "${X_EXTRA_ARGS[@]}" \ "${IMAGE_NAME}" diff --git a/test/helper/tls.bash b/test/helper/tls.bash index 4987a66f..bb107a4f 100644 --- a/test/helper/tls.bash +++ b/test/helper/tls.bash @@ -1,21 +1,17 @@ #!/bin/bash -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/common" -# Helper methods for testing TLS. # `_should_*` methods are useful for common high-level functionality. - # ? --------------------------------------------- Negotiate TLS - # For certs actually provisioned from LetsEncrypt the Root CA cert should not need to be provided, # as it would already be available by default in `/etc/ssl/certs`, requiring only the cert chain (fullchain.pem). function _should_succesfully_negotiate_tls() { local FQDN=${1} - local CONTAINER_NAME=${2:-${TEST_NAME}} # shellcheck disable=SC2031 - local CA_CERT=${3:-${TEST_CA_CERT}} + local CA_CERT=${2:-${TEST_CA_CERT}} # Postfix and Dovecot are ready: wait_for_smtp_port_in_container_to_respond "${CONTAINER_NAME}" @@ -36,9 +32,8 @@ function _should_succesfully_negotiate_tls() { function _negotiate_tls() { local FQDN=${1} local PORT=${2} - local CONTAINER_NAME=${3:-${TEST_NAME}} # shellcheck disable=SC2031 - local CA_CERT=${4:-${TEST_CA_CERT}} + local CA_CERT=${3:-${TEST_CA_CERT}} local CMD_OPENSSL_VERIFY CMD_OPENSSL_VERIFY=$(_generate_openssl_cmd "${PORT}") @@ -83,7 +78,6 @@ function _generate_openssl_cmd() { echo "${CMD_OPENSSL} ${EXTRA_ARGS} 2>/dev/null" } - # ? --------------------------------------------- Verify FQDN function _get_fqdn_match_query() { @@ -115,9 +109,8 @@ function escape_fqdn() { function _get_fqdns_for_cert() { local FQDN=${1} local PORT=${2:-'25'} - local CONTAINER_NAME=${3:-${TEST_NAME}} # shellcheck disable=SC2031 - local CA_CERT=${4:-${TEST_CA_CERT}} + local CA_CERT=${3:-${TEST_CA_CERT}} # `-servername` is for SNI, where the port may be for a service that serves multiple certs, # and needs a specific FQDN to return the correct cert. Such as a reverse-proxy. From 2ec6c4abc03055c41373d7f9e7c40e580bbd164d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 3 Jan 2023 18:58:09 +1300 Subject: [PATCH 3/5] tests: Adjust parallel tests - The usual serial to parallel test conversion to utilize the `setup.bash` common setup structure, and adding a `TEST_PREFIX` var for each test case to leverage. - Standardize on parallel test naming conventions for variables / values. - More consistent use of `bash -c` instead of `/bin/bash -c` or `/bin/sh -c`. - Using the `_run_in_container` helper instead of `run docker exec ${CONTAINER_NAME}`. - Updates tests to use the `check_if_process_is_running` helper. --- chore: Revise inline docs for the `ssl_letsencrypt` test - Moves the override to be in closer proximity to the `initial_setup` call, and better communicates the intent to override. - Removes top comment block that is no longer providing value or correct information to maintainers. - Revised `acme.json` test case inline doc comments. --- test/helper/common.bash | 4 +- .../parallel/set1/default_relay_host.bats | 10 +- .../parallel/set1/spam_virus/clamav.bats | 8 +- .../disabled_clamav_spamassassin.bats | 58 ++-- .../tests/parallel/set1/spam_virus/dnsbl.bats | 70 ++--- .../parallel/set1/spam_virus/fail2ban.bats | 255 ++++++++++-------- .../set1/spam_virus/postgrey_enabled.bats | 92 +++---- .../parallel/set1/spam_virus/postscreen.bats | 58 ++-- .../set1/spam_virus/spam_bounced.bats | 4 +- .../set1/spam_virus/spam_junk_folder.bats | 84 +++--- .../set1/spam_virus/undef_spam_subject.bats | 89 +++--- test/tests/parallel/set2/template.bats | 2 +- .../parallel/set2/tls/tls_cipherlists.bats | 242 +++++++++-------- .../parallel/set2/tls/tls_letsencrypt.bats | 115 ++++---- test/tests/parallel/set2/tls/tls_manual.bats | 104 +++---- .../parallel/set3/dovecot_inet_protocol.bats | 17 +- .../tests/parallel/set3/helper-functions.bats | 2 +- 17 files changed, 623 insertions(+), 591 deletions(-) diff --git a/test/helper/common.bash b/test/helper/common.bash index 1f38b132..2eba1086 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -30,10 +30,10 @@ function _default_teardown() { # @param ${1} program name [REQUIRED] # @param ${2} container name [IF UNSET: ${CONTAINER_NAME}] -function _check_if_process_is_running() { +function check_if_process_is_running() { local PROGRAM_NAME=${1:?Program name must be provided explicitly} local CONTAINER_NAME=${2:-${CONTAINER_NAME}} - _run_in_container_explicit "${CONTAINER_NAME}" pgrep "${PROGRAM_NAME}" + docker exec "${CONTAINER_NAME}" pgrep "${PROGRAM_NAME}" } # ------------------------------------------------------------------- diff --git a/test/tests/parallel/set1/default_relay_host.bats b/test/tests/parallel/set1/default_relay_host.bats index ead7925e..9b79361a 100644 --- a/test/tests/parallel/set1/default_relay_host.bats +++ b/test/tests/parallel/set1/default_relay_host.bats @@ -1,15 +1,15 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" -export TEST_NAME_PREFIX='default relay host:' -export CONTAINER_NAME='dms-test-default_relay_host' +TEST_NAME_PREFIX='[Relay] ENV:' +CONTAINER_NAME='dms-test_default-relay-host' function setup_file() { init_with_defaults local CUSTOM_SETUP_ARGUMENTS=( - --env DEFAULT_RELAY_HOST=default.relay.host.invalid:25 \ - --env PERMIT_DOCKER=host \ + --env DEFAULT_RELAY_HOST=default.relay.host.invalid:25 + --env PERMIT_DOCKER=host ) common_container_setup 'CUSTOM_SETUP_ARGUMENTS' @@ -17,7 +17,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } -@test "${TEST_NAME_PREFIX} default relay host is added to main.cf" { +@test "${TEST_NAME_PREFIX} 'DEFAULT_RELAY_HOST' should configure 'main.cf:relayhost'" { _run_in_container bash -c 'grep -e "^relayhost =" /etc/postfix/main.cf' assert_output 'relayhost = default.relay.host.invalid:25' } diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 42450926..1a894750 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -2,7 +2,7 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" TEST_NAME_PREFIX='ClamAV:' -CONTAINER_NAME='dms-test-clamav' +CONTAINER_NAME='dms-test_clamav' function setup_file() { init_with_defaults @@ -35,7 +35,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test "${TEST_NAME_PREFIX} process clamd is running" { - _run_in_container bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" + run check_if_process_is_running 'clamd' assert_success } @@ -66,6 +66,8 @@ function teardown_file() { _default_teardown ; } } @test "${TEST_NAME_PREFIX} process clamd restarts when killed" { - _run_in_container bash -c "pkill clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" + _run_in_container pkill 'clamd' assert_success + + run_until_success_or_timeout 10 check_if_process_is_running 'clamd' } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 83e6a398..23fd00c0 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -1,48 +1,50 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -setup_file() { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) +TEST_NAME_PREFIX='[ClamAV + SA] (disabled):' +CONTAINER_NAME='dms-test_clamav-spamassasin_disabled' - docker run --rm -d --name mail_disabled_clamav_spamassassin \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_CLAMAV=0 \ - -e ENABLE_SPAMASSASSIN=0 \ - -e AMAVIS_LOGLEVEL=2 \ - -h mail.my-domain.com -t "${NAME}" +function setup_file() { + init_with_defaults - # TODO: find a better way to know when we have waited long enough - # for ClamAV to should have come up, if it were enabled - wait_for_smtp_port_in_container mail_disabled_clamav_spamassassin - docker exec mail_disabled_clamav_spamassassin /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt" + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_CLAMAV=0 + --env ENABLE_SPAMASSASSIN=0 + --env AMAVIS_LOGLEVEL=2 + ) + + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" + + _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt" + assert_success + wait_for_empty_mail_queue_in_container "${CONTAINER_NAME}" } -teardown_file() { - docker rm -f mail_disabled_clamav_spamassassin -} +function teardown_file() { _default_teardown ; } -@test "checking process: ClamAV (ClamAV disabled by ENABLED_CLAMAV=0)" { - run docker exec mail_disabled_clamav_spamassassin /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" +@test "${TEST_NAME_PREFIX} ClamAV - should be disabled by ENV 'ENABLED_CLAMAV=0'" { + run check_if_process_is_running 'clamd' assert_failure } -@test "checking spamassassin: should not be listed in amavis when disabled" { - run docker exec mail_disabled_clamav_spamassassin /bin/sh -c "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'" +@test "${TEST_NAME_PREFIX} SA - Amavis integration should not be active" { + _run_in_container /bin/sh -c "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'" assert_success } -@test "checking ClamAV: should not be listed in amavis when disabled" { - run docker exec mail_disabled_clamav_spamassassin grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log +@test "${TEST_NAME_PREFIX} ClamAV - Amavis integration should not be active" { + _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log assert_failure } -@test "checking ClamAV: should not be called when disabled" { - run docker exec mail_disabled_clamav_spamassassin grep -i 'connect to /var/run/clamav/clamd.ctl failed' /var/log/mail/mail.log +@test "${TEST_NAME_PREFIX} SA should not be called" { + _run_in_container grep -i 'connect to /var/run/clamav/clamd.ctl failed' /var/log/mail/mail.log assert_failure } -@test "checking restart of process: ClamAV (ClamAV disabled by ENABLED_CLAMAV=0)" { - run docker exec mail_disabled_clamav_spamassassin /bin/bash -c "pkill -f clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" +@test "${TEST_NAME_PREFIX} ClamAV process should not be restarted when killed" { + _run_in_container /bin/bash -c "pkill -f clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" assert_failure } diff --git a/test/tests/parallel/set1/spam_virus/dnsbl.bats b/test/tests/parallel/set1/spam_virus/dnsbl.bats index d4c3a5d4..11cc7ae2 100644 --- a/test/tests/parallel/set1/spam_virus/dnsbl.bats +++ b/test/tests/parallel/set1/spam_virus/dnsbl.bats @@ -1,61 +1,61 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -CONTAINER="mail_dnsbl_enabled" -CONTAINER2="mail_dnsbl_disabled" +TEST_NAME_PREFIX='DNSBLs:' + +CONTAINER1_NAME='dms-test_dnsbl_enabled' +CONTAINER2_NAME='dms-test_dnsbl_disabled' function setup_file() { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container . "${CONTAINER}") + local CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_DNSBL=1 + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" - docker run --rm -d --name "${CONTAINER}" \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -e ENABLE_DNSBL=1 \ - -h mail.my-domain.com \ - -t "${NAME}" + local CONTAINER_NAME=${CONTAINER2_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_DNSBL=0 + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" +} - docker run --rm -d --name "${CONTAINER2}" \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -e ENABLE_DNSBL=0 \ - -h mail.my-domain.com \ - -t "${NAME}" - - wait_for_smtp_port_in_container "${CONTAINER}" - wait_for_smtp_port_in_container "${CONTAINER2}" +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } # ENABLE_DNSBL=1 -@test "checking enabled postfix DNS block list zen.spamhaus.org" { - run docker exec "${CONTAINER}" postconf smtpd_recipient_restrictions +@test "${TEST_NAME_PREFIX} (enabled) Postfix DNS block list zen.spamhaus.org" { + run docker exec "${CONTAINER1_NAME}" postconf smtpd_recipient_restrictions assert_output --partial 'reject_rbl_client zen.spamhaus.org' } -@test "checking enabled postscreen DNS block lists --> postscreen_dnsbl_action" { - run docker exec "${CONTAINER}" postconf postscreen_dnsbl_action +@test "${TEST_NAME_PREFIX} (enabled) Postscreen DNS block lists -> postscreen_dnsbl_action" { + run docker exec "${CONTAINER1_NAME}" postconf postscreen_dnsbl_action assert_output 'postscreen_dnsbl_action = enforce' } -@test "checking enabled postscreen DNS block lists --> postscreen_dnsbl_sites" { - run docker exec "${CONTAINER}" postconf postscreen_dnsbl_sites +@test "${TEST_NAME_PREFIX} (enabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" { + run docker exec "${CONTAINER1_NAME}" postconf postscreen_dnsbl_sites assert_output 'postscreen_dnsbl_sites = zen.spamhaus.org=127.0.0.[2..11]*3 bl.mailspike.net=127.0.0.[2;14;13;12;11;10] b.barracudacentral.org*2 bl.spameatingmonkey.net=127.0.0.2 dnsbl.sorbs.net psbl.surriel.com list.dnswl.org=127.0.[0..255].0*-2 list.dnswl.org=127.0.[0..255].1*-3 list.dnswl.org=127.0.[0..255].[2..3]*-4' } # ENABLE_DNSBL=0 -@test "checking disabled postfix DNS block list zen.spamhaus.org" { - run docker exec "${CONTAINER2}" postconf smtpd_recipient_restrictions +@test "${TEST_NAME_PREFIX} (disabled) Postfix DNS block list zen.spamhaus.org" { + run docker exec "${CONTAINER2_NAME}" postconf smtpd_recipient_restrictions refute_output --partial 'reject_rbl_client zen.spamhaus.org' } -@test "checking disabled postscreen DNS block lists --> postscreen_dnsbl_action" { - run docker exec "${CONTAINER2}" postconf postscreen_dnsbl_action +@test "${TEST_NAME_PREFIX} (disabled) Postscreen DNS block lists -> postscreen_dnsbl_action" { + run docker exec "${CONTAINER2_NAME}" postconf postscreen_dnsbl_action assert_output 'postscreen_dnsbl_action = ignore' } -@test "checking disabled postscreen DNS block lists --> postscreen_dnsbl_sites" { - run docker exec "${CONTAINER2}" postconf postscreen_dnsbl_sites +@test "${TEST_NAME_PREFIX} (disabled) Postscreen DNS block lists -> postscreen_dnsbl_sites" { + run docker exec "${CONTAINER2_NAME}" postconf postscreen_dnsbl_sites assert_output 'postscreen_dnsbl_sites =' } - -# cleanup -function teardown_file() { - docker rm -f "${CONTAINER}" "${CONTAINER2}" -} diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index a3f927d7..29a5e620 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -1,88 +1,90 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +TEST_NAME_PREFIX='Fail2Ban:' +CONTAINER1_NAME='dms-test_fail2ban' +CONTAINER2_NAME='dms-test_fail2ban_fail-auth-mailer' + +function get_container2_ip() { + docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${CONTAINER2_NAME}" +} function setup_file() { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) - docker run --rm -d --name mail_fail2ban \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_FAIL2BAN=1 \ - -e POSTSCREEN_ACTION=ignore \ - --cap-add=NET_ADMIN \ - --hostname mail.my-domain.com \ - --tty \ - --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" \ - "${NAME}" + export CONTAINER_NAME + + CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_FAIL2BAN=1 + --env POSTSCREEN_ACTION=ignore + --cap-add=NET_ADMIN + --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" # Create a container which will send wrong authentications and should get banned - docker run --name fail-auth-mailer \ - -e MAIL_FAIL2BAN_IP="$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' mail_fail2ban)" \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test \ - -d "${NAME}" \ - tail -f /var/log/faillog + CONTAINER_NAME=${CONTAINER2_NAME} + local CUSTOM_SETUP_ARGUMENTS=(--env MAIL_FAIL2BAN_IP="$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CONTAINER1_NAME})") + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - wait_for_finished_setup_in_container mail_fail2ban + # Set default implicit container fallback for helpers: + CONTAINER_NAME=${CONTAINER1_NAME} } function teardown_file() { - docker rm -f mail_fail2ban fail-auth-mailer + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } -# -# processes -# - -@test "checking process: fail2ban (fail2ban server enabled)" { - run docker exec mail_fail2ban /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/python3 /usr/bin/fail2ban-server'" +@test "${TEST_NAME_PREFIX} Fail2Ban is running" { + run check_if_process_is_running 'fail2ban-server' assert_success } -# -# fail2ban -# +@test "${TEST_NAME_PREFIX} localhost is not banned because ignored" { + _run_in_container fail2ban-client status postfix-sasl + assert_success + refute_output --regexp '.*IP list:.*127\.0\.0\.1.*' -@test "checking fail2ban: localhost is not banned because ignored" { - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client status postfix-sasl | grep 'IP list:.*127.0.0.1'" - assert_failure - run docker exec mail_fail2ban /bin/sh -c "grep 'ignoreip = 127.0.0.1/8' /etc/fail2ban/jail.conf" + _run_in_container grep 'ignoreip = 127.0.0.1/8' /etc/fail2ban/jail.conf assert_success } -@test "checking fail2ban: fail2ban-fail2ban.cf overrides" { - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client get loglevel | grep DEBUG" +@test "${TEST_NAME_PREFIX} fail2ban-fail2ban.cf overrides" { + _run_in_container fail2ban-client get loglevel assert_success + assert_output --partial 'DEBUG' } -@test "checking fail2ban: fail2ban-jail.cf overrides" { - FILTERS=(dovecot postfix postfix-sasl) - - for FILTER in "${FILTERS[@]}"; do - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client get ${FILTER} bantime" +@test "${TEST_NAME_PREFIX} fail2ban-jail.cf overrides" { + for FILTER in 'dovecot' 'postfix' 'postfix-sasl' + do + _run_in_container fail2ban-client get "${FILTER}" bantime assert_output 1234 - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client get ${FILTER} findtime" + _run_in_container fail2ban-client get "${FILTER}" findtime assert_output 321 - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client get ${FILTER} maxretry" + _run_in_container fail2ban-client get "${FILTER}" maxretry assert_output 2 - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client -d | grep -F \"['set', 'dovecot', 'addaction', 'nftables-multiport']\"" - assert_output "['set', 'dovecot', 'addaction', 'nftables-multiport']" - - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client -d | grep -F \"['set', 'postfix', 'addaction', 'nftables-multiport']\"" - assert_output "['set', 'postfix', 'addaction', 'nftables-multiport']" - - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client -d | grep -F \"['set', 'postfix-sasl', 'addaction', 'nftables-multiport']\"" - assert_output "['set', 'postfix-sasl', 'addaction', 'nftables-multiport']" + _run_in_container fail2ban-client -d + assert_output --partial "['set', 'dovecot', 'addaction', 'nftables-multiport']" + assert_output --partial "['set', 'postfix', 'addaction', 'nftables-multiport']" + assert_output --partial "['set', 'postfix-sasl', 'addaction', 'nftables-multiport']" done } -@test "checking fail2ban: ban ip on multiple failed login" { - # can't pipe the file as usual due to postscreen. (respecting postscreen_greet_wait time and talking in turn): +# NOTE: This test case is fragile if other test cases were to be run concurrently +@test "${TEST_NAME_PREFIX} ban ip on multiple failed login" { + # can't pipe the file as usual due to postscreen + # respecting postscreen_greet_wait time and talking in turn): + # shellcheck disable=SC1004 for _ in {1,2} do - docker exec fail-auth-mailer /bin/bash -c \ + docker exec "${CONTAINER2_NAME}" /bin/bash -c \ 'exec 3<>/dev/tcp/${MAIL_FAIL2BAN_IP}/25 && \ while IFS= read -r cmd; do \ head -1 <&3; \ @@ -93,108 +95,123 @@ function teardown_file() { sleep 5 - FAIL_AUTH_MAILER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' fail-auth-mailer) - # Checking that FAIL_AUTH_MAILER_IP is banned in mail_fail2ban - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client status postfix-sasl | grep '${FAIL_AUTH_MAILER_IP}'" + # Checking that CONTAINER2_IP is banned in "${CONTAINER1_NAME}" + CONTAINER2_IP=$(get_container2_ip) + _run_in_container fail2ban-client status postfix-sasl assert_success + assert_output --partial "${CONTAINER2_IP}" - # Checking that FAIL_AUTH_MAILER_IP is banned by nftables - run docker exec mail_fail2ban /bin/sh -c "nft list set inet f2b-table addr-set-postfix-sasl" - assert_output --partial "elements = { ${FAIL_AUTH_MAILER_IP} }" + # Checking that CONTAINER2_IP is banned by nftables + _run_in_container bash -c 'nft list set inet f2b-table addr-set-postfix-sasl' + assert_success + assert_output --partial "elements = { ${CONTAINER2_IP} }" } -@test "checking fail2ban: unban ip works" { - FAIL_AUTH_MAILER_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' fail-auth-mailer) - docker exec mail_fail2ban fail2ban-client set postfix-sasl unbanip "${FAIL_AUTH_MAILER_IP}" - +@test "${TEST_NAME_PREFIX} unban ip works" { + CONTAINER2_IP=$(get_container2_ip) + _run_in_container fail2ban-client set postfix-sasl unbanip "${CONTAINER2_IP}" + assert_success sleep 5 - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client status postfix-sasl | grep 'IP list:.*${FAIL_AUTH_MAILER_IP}'" - assert_failure + # Checking that CONTAINER2_IP is unbanned in "${CONTAINER1_NAME}" + _run_in_container fail2ban-client status postfix-sasl + assert_success + refute_output --partial "${CONTAINER2_IP}" - # Checking that FAIL_AUTH_MAILER_IP is unbanned by nftables - run docker exec mail_fail2ban /bin/sh -c "nft list set inet f2b-table addr-set-postfix-sasl" - refute_output --partial "${FAIL_AUTH_MAILER_IP}" + # Checking that CONTAINER2_IP is unbanned by nftables + _run_in_container bash -c 'nft list set inet f2b-table addr-set-postfix-sasl' + refute_output --partial "${CONTAINER2_IP}" } -@test "checking fail2ban ban" { - # Ban single IP address - run docker exec mail_fail2ban fail2ban ban 192.0.66.7 +@test "${TEST_NAME_PREFIX} bans work properly (single IP)" { + _run_in_container fail2ban ban 192.0.66.7 assert_success - assert_output "Banned custom IP: 1" + assert_output 'Banned custom IP: 1' - run docker exec mail_fail2ban fail2ban + _run_in_container fail2ban assert_success - assert_output --regexp "Banned in custom:.*192\.0\.66\.7" + assert_output --regexp 'Banned in custom:.*192\.0\.66\.7' - run docker exec mail_fail2ban nft list set inet f2b-table addr-set-custom + _run_in_container nft list set inet f2b-table addr-set-custom assert_success - assert_output --partial "elements = { 192.0.66.7 }" + assert_output --partial 'elements = { 192.0.66.7 }' - run docker exec mail_fail2ban fail2ban unban 192.0.66.7 + _run_in_container fail2ban unban 192.0.66.7 assert_success - assert_output --partial "Unbanned IP from custom: 1" + assert_output --partial 'Unbanned IP from custom: 1' - run docker exec mail_fail2ban nft list set inet f2b-table addr-set-custom - refute_output --partial "192.0.66.7" - - # Ban IP network - run docker exec mail_fail2ban fail2ban ban 192.0.66.0/24 - assert_success - assert_output "Banned custom IP: 1" - - run docker exec mail_fail2ban fail2ban - assert_success - assert_output --regexp "Banned in custom:.*192\.0\.66\.0/24" - - run docker exec mail_fail2ban nft list set inet f2b-table addr-set-custom - assert_success - assert_output --partial "elements = { 192.0.66.0/24 }" - - run docker exec mail_fail2ban fail2ban unban 192.0.66.0/24 - assert_success - assert_output --partial "Unbanned IP from custom: 1" - - run docker exec mail_fail2ban nft list set inet f2b-table addr-set-custom - refute_output --partial "192.0.66.0/24" + _run_in_container nft list set inet f2b-table addr-set-custom + refute_output --partial '192.0.66.7' } -@test "checking FAIL2BAN_BLOCKTYPE is really set to drop" { - run docker exec mail_fail2ban bash -c 'nft list table inet f2b-table' +@test "${TEST_NAME_PREFIX} bans work properly (subnet)" { + _run_in_container fail2ban ban 192.0.66.0/24 + assert_success + assert_output 'Banned custom IP: 1' + + _run_in_container fail2ban + assert_success + assert_output --regexp 'Banned in custom:.*192\.0\.66\.0/24' + + _run_in_container nft list set inet f2b-table addr-set-custom + assert_success + assert_output --partial 'elements = { 192.0.66.0/24 }' + + _run_in_container fail2ban unban 192.0.66.0/24 + assert_success + assert_output --partial 'Unbanned IP from custom: 1' + + _run_in_container nft list set inet f2b-table addr-set-custom + refute_output --partial '192.0.66.0/24' +} + +@test "${TEST_NAME_PREFIX} FAIL2BAN_BLOCKTYPE is really set to drop" { + # ban IPs here manually so we can be sure something is inside the jails + for JAIL in dovecot postfix-sasl custom; do + _run_in_container fail2ban-client set "${JAIL}" banip 192.33.44.55 + assert_success + done + + _run_in_container nft list table inet f2b-table assert_success assert_output --partial 'tcp dport { 110, 143, 465, 587, 993, 995, 4190 } ip saddr @addr-set-dovecot drop' assert_output --partial 'tcp dport { 25, 110, 143, 465, 587, 993, 995 } ip saddr @addr-set-postfix-sasl drop' assert_output --partial 'tcp dport { 25, 110, 143, 465, 587, 993, 995, 4190 } ip saddr @addr-set-custom drop' + + # unban the IPs previously banned to get a clean state again + for JAIL in dovecot postfix-sasl custom; do + _run_in_container fail2ban-client set "${JAIL}" unbanip 192.33.44.55 + assert_success + done } -@test "checking setup.sh: setup.sh fail2ban" { - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client set dovecot banip 192.0.66.4" - run docker exec mail_fail2ban /bin/sh -c "fail2ban-client set dovecot banip 192.0.66.5" +@test "${TEST_NAME_PREFIX} setup.sh fail2ban" { + _run_in_container fail2ban-client set dovecot banip 192.0.66.4 + _run_in_container fail2ban-client set dovecot banip 192.0.66.5 sleep 10 - run ./setup.sh -c mail_fail2ban fail2ban + # Originally: run ./setup.sh -c "${CONTAINER1_NAME}" fail2ban + _run_in_container setup fail2ban assert_output --regexp '^Banned in dovecot:.*192\.0\.66\.4' assert_output --regexp '^Banned in dovecot:.*192\.0\.66\.5' - run ./setup.sh -c mail_fail2ban fail2ban unban 192.0.66.4 + _run_in_container setup fail2ban unban 192.0.66.4 assert_output --partial "Unbanned IP from dovecot: 1" - run ./setup.sh -c mail_fail2ban fail2ban - assert_output --regexp "^Banned in dovecot:.*192\.0\.66\.5" + _run_in_container setup fail2ban + assert_output --regexp '^Banned in dovecot:.*192\.0\.66\.5' - run ./setup.sh -c mail_fail2ban fail2ban unban 192.0.66.5 - assert_output --partial "Unbanned IP from dovecot: 1" + _run_in_container setup fail2ban unban 192.0.66.5 + assert_output --partial 'Unbanned IP from dovecot: 1' - run ./setup.sh -c mail_fail2ban fail2ban unban - assert_output --partial "You need to specify an IP address: Run" + _run_in_container setup fail2ban unban + assert_output --partial 'You need to specify an IP address: Run' } -# -# supervisor -# - -@test "checking restart of process: fail2ban (fail2ban server enabled)" { - run docker exec mail_fail2ban /bin/bash -c "pkill fail2ban && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/bin/python3 /usr/bin/fail2ban-server'" +@test "${TEST_NAME_PREFIX} restart of Fail2Ban" { + _run_in_container pkill fail2ban assert_success + + run_until_success_or_timeout 10 check_if_process_is_running 'fail2ban-server' } diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 9fd4fb12..4244d7d5 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -1,92 +1,92 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +TEST_NAME_PREFIX='Postgrey (enabled):' +CONTAINER_NAME='dms-test_postgrey_enabled' function setup_file() { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_DNSBL=1 + --env ENABLE_POSTGREY=1 + --env PERMIT_DOCKER=container + --env POSTGREY_AUTO_WHITELIST_CLIENTS=5 + --env POSTGREY_DELAY=15 + --env POSTGREY_MAX_AGE=35 + --env POSTGREY_TEXT="Delayed by Postgrey" + ) - docker run -d --name mail_with_postgrey \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_DNSBL=1 \ - -e ENABLE_POSTGREY=1 \ - -e PERMIT_DOCKER=container \ - -e POSTGREY_AUTO_WHITELIST_CLIENTS=5 \ - -e POSTGREY_DELAY=15 \ - -e POSTGREY_MAX_AGE=35 \ - -e POSTGREY_TEXT="Delayed by Postgrey" \ - -h mail.my-domain.com -t "${NAME}" + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - # using postfix availability as start indicator, this might be insufficient for postgrey - wait_for_smtp_port_in_container mail_with_postgrey + # Postfix needs to be ready on port 25 for nc usage below: + wait_for_smtp_port_in_container "${CONTAINER_NAME}" } -function teardown_file() { - docker rm -f mail_with_postgrey -} +function teardown_file() { _default_teardown ; } -@test "checking postgrey: /etc/postfix/main.cf correctly edited" { - run docker exec mail_with_postgrey /bin/bash -c "grep -F 'zen.spamhaus.org=127.0.0.[2..11], check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf | wc -l" +@test "${TEST_NAME_PREFIX} /etc/postfix/main.cf correctly edited" { + _run_in_container bash -c "grep -F 'zen.spamhaus.org=127.0.0.[2..11], check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf | wc -l" assert_success assert_output 1 } -@test "checking postgrey: /etc/default/postgrey correctly edited and has the default values" { - run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_OPTS=\"--inet=127.0.0.1:10023 --delay=15 --max-age=35 --auto-whitelist-clients=5\"$' /etc/default/postgrey | wc -l" +@test "${TEST_NAME_PREFIX} /etc/default/postgrey correctly edited and has the default values" { + _run_in_container bash -c "grep '^POSTGREY_OPTS=\"--inet=127.0.0.1:10023 --delay=15 --max-age=35 --auto-whitelist-clients=5\"$' /etc/default/postgrey | wc -l" assert_success assert_output 1 - run docker exec mail_with_postgrey /bin/bash -c "grep '^POSTGREY_TEXT=\"Delayed by Postgrey\"$' /etc/default/postgrey | wc -l" + _run_in_container bash -c "grep '^POSTGREY_TEXT=\"Delayed by Postgrey\"$' /etc/default/postgrey | wc -l" assert_success assert_output 1 } -@test "checking process: postgrey (postgrey server enabled)" { - run docker exec mail_with_postgrey /bin/bash -c "ps aux --forest | grep -v grep | grep 'postgrey'" +@test "${TEST_NAME_PREFIX} Postgrey is running" { + run check_if_process_is_running 'postgrey' assert_success } -@test "checking postgrey: there should be a log entry about a new greylisted e-mail user@external.tld in /var/log/mail/mail.log" { +@test "${TEST_NAME_PREFIX} there should be a log entry about a new greylisted e-mail user@external.tld in /var/log/mail/mail.log" { #editing the postfix config in order to ensure that postgrey handles the test e-mail. The other spam checks at smtpd_recipient_restrictions would interfere with it. - run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf" - run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' /etc/postfix/main.cf" - run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' /etc/postfix/main.cf" - run docker exec mail_with_postgrey /bin/sh -c "sed -ie 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' /etc/postfix/main.cf" + _run_in_container bash -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf" + _run_in_container bash -c "sed -ie 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' /etc/postfix/main.cf" + _run_in_container bash -c "sed -ie 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' /etc/postfix/main.cf" + _run_in_container bash -c "sed -ie 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' /etc/postfix/main.cf" + _run_in_container postfix reload - run docker exec mail_with_postgrey /bin/sh -c "/etc/init.d/postfix reload" - run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" + _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" sleep 5 #ensure that the information has been written into the log - run docker exec mail_with_postgrey /bin/bash -c "grep -i 'action=greylist.*user@external\.tld' /var/log/mail/mail.log | wc -l" + _run_in_container bash -c "grep -i 'action=greylist.*user@external\.tld' /var/log/mail/mail.log | wc -l" assert_success assert_output 1 } -@test "checking postgrey: there should be a log entry about the retried and passed e-mail user@external.tld in /var/log/mail/mail.log" { +@test "${TEST_NAME_PREFIX} there should be a log entry about the retried and passed e-mail user@external.tld in /var/log/mail/mail.log" { sleep 20 #wait 20 seconds so that postgrey would accept the message - run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" + _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" sleep 8 - run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=triplet found.*user@external\.tld' /var/log/mail/mail.log | wc -l" + _run_in_container bash -c "grep -i 'action=pass, reason=triplet found.*user@external\.tld' /var/log/mail/mail.log | wc -l" assert_success assert_output 1 } -@test "checking postgrey: there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" { - run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt" - run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" +@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" { + _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt" + _run_in_container bash -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" assert_success assert_output 1 } -@test "checking postgrey: there should be a log entry about the whitelisted local and passed e-mail user@whitelistlocal.tld in /var/log/mail/mail.log" { - run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_local.txt" - run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" +@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted local and passed e-mail user@whitelistlocal.tld in /var/log/mail/mail.log" { + _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_local.txt" + _run_in_container bash -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" assert_success assert_output 1 } -@test "checking postgrey: there should be a log entry about the whitelisted recipient user2@otherdomain.tld in /var/log/mail/mail.log" { - run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt" - run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=recipient whitelist' /var/log/mail/mail.log | wc -l" +@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted recipient user2@otherdomain.tld in /var/log/mail/mail.log" { + _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt" + _run_in_container bash -c "grep -i 'action=pass, reason=recipient whitelist' /var/log/mail/mail.log | wc -l" assert_success assert_output 1 } diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index aa2f958a..e44c5bb1 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -1,44 +1,46 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -setup() { - # Getting mail container IP - MAIL_POSTSCREEN_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' mail_postscreen) +TEST_NAME_PREFIX='Postscreen:' +CONTAINER1_NAME='dms-test_postscreen_enforce' +CONTAINER2_NAME='dms-test_postscreen_sender' + +function setup() { + MAIL_POSTSCREEN_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${CONTAINER1_NAME}") } -setup_file() { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) +function setup_file() { + local CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env POSTSCREEN_ACTION=enforce + --cap-add=NET_ADMIN + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" - docker run -d --name mail_postscreen \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e POSTSCREEN_ACTION=enforce \ - --cap-add=NET_ADMIN \ - -h mail.my-domain.com -t "${NAME}" - - docker run --name mail_postscreen_sender \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -d "${NAME}" \ - tail -f /var/log/faillog - - wait_for_smtp_port_in_container mail_postscreen + local CONTAINER_NAME=${CONTAINER2_NAME} + init_with_defaults + common_container_setup + wait_for_smtp_port_in_container "${CONTAINER_NAME}" } -teardown_file() { - docker rm -f mail_postscreen mail_postscreen_sender +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } -@test "checking postscreen: talk too fast" { - docker exec mail_postscreen_sender /bin/sh -c "nc ${MAIL_POSTSCREEN_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt" +@test "${TEST_NAME_PREFIX} talk too fast" { + run docker exec "${CONTAINER2_NAME}" /bin/sh -c "nc ${MAIL_POSTSCREEN_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt" + assert_success - repeat_until_success_or_timeout 10 run docker exec mail_postscreen grep 'COMMAND PIPELINING' /var/log/mail/mail.log + repeat_until_success_or_timeout 10 run docker exec "${CONTAINER1_NAME}" grep 'COMMAND PIPELINING' /var/log/mail/mail.log assert_success } -@test "checking postscreen: positive test (respecting postscreen_greet_wait time and talking in turn)" { +@test "${TEST_NAME_PREFIX} positive test (respecting postscreen_greet_wait time and talking in turn)" { for _ in {1,2}; do # shellcheck disable=SC1004 - docker exec mail_postscreen_sender /bin/bash -c \ + docker exec "${CONTAINER2_NAME}" /bin/bash -c \ 'exec 3<>/dev/tcp/'"${MAIL_POSTSCREEN_IP}"'/25 && \ while IFS= read -r cmd; do \ head -1 <&3; \ @@ -47,6 +49,6 @@ teardown_file() { done < "/tmp/docker-mailserver-test/auth/smtp-auth-login.txt"' done - repeat_until_success_or_timeout 10 run docker exec mail_postscreen grep 'PASS NEW ' /var/log/mail/mail.log + repeat_until_success_or_timeout 10 run docker exec "${CONTAINER1_NAME}" grep 'PASS NEW ' /var/log/mail/mail.log assert_success } diff --git a/test/tests/parallel/set1/spam_virus/spam_bounced.bats b/test/tests/parallel/set1/spam_virus/spam_bounced.bats index 321d2981..1931149e 100644 --- a/test/tests/parallel/set1/spam_virus/spam_bounced.bats +++ b/test/tests/parallel/set1/spam_virus/spam_bounced.bats @@ -1,8 +1,8 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" -TEST_NAME_PREFIX='spam (Amavis):' -CONTAINER_NAME='dms-test-spam_bounced' +TEST_NAME_PREFIX='Spam bounced:' +CONTAINER_NAME='dms-test_spam-bounced' function setup_file() { init_with_defaults diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index e335d06f..b69f774d 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -1,67 +1,67 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +TEST_NAME_PREFIX='Spam junk folder:' +CONTAINER1_NAME='dms-test_spam-junk-folder_1' +CONTAINER2_NAME='dms-test_spam-junk-folder_2' + +function teardown() { _default_teardown ; } # Test case # --------- -# When SPAMASSASSIN_SPAM_TO_INBOX=1, spam messages must be delivered and eventually (MOVE_SPAM_TO_JUNK=1) moved to the Junk folder. +# When SPAMASSASSIN_SPAM_TO_INBOX=1, spam messages must be delivered +# and eventually (MOVE_SPAM_TO_JUNK=1) moved to the Junk folder. -@test "checking amavis: spam message is delivered and moved to the Junk folder (MOVE_SPAM_TO_JUNK=1)" { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container . mail_spam_moved_junk) - - docker run -d --name mail_spam_moved_junk \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_SPAMASSASSIN=1 \ - -e MOVE_SPAM_TO_JUNK=1 \ - -e PERMIT_DOCKER=container \ - -e SA_SPAM_SUBJECT="SPAM: " \ - -e SPAMASSASSIN_SPAM_TO_INBOX=1 \ - -h mail.my-domain.com -t "${NAME}" - - teardown() { docker rm -f mail_spam_moved_junk; } - - wait_for_smtp_port_in_container mail_spam_moved_junk +@test "${TEST_NAME_PREFIX} (Amavis) spam message delivered & moved to Junk folder" { + export CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env MOVE_SPAM_TO_JUNK=1 + --env PERMIT_DOCKER=container + --env SA_SPAM_SUBJECT="SPAM: " + --env SPAMASSASSIN_SPAM_TO_INBOX=1 + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" # send a spam message - run docker exec mail_spam_moved_junk /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt" + _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt" assert_success # message will be added to a queue with varying delay until amavis receives it - run repeat_until_success_or_timeout 60 sh -c "docker logs mail_spam_moved_junk | grep 'Passed SPAM {RelayedTaggedInbound,Quarantined}'" + run repeat_until_success_or_timeout 60 bash -c "docker logs ${CONTAINER_NAME} | grep 'Passed SPAM {RelayedTaggedInbound,Quarantined}'" assert_success # spam moved to Junk folder - run repeat_until_success_or_timeout 20 sh -c "docker exec mail_spam_moved_junk sh -c 'grep \"Subject: SPAM: \" /var/mail/localhost.localdomain/user1/.Junk/new/ -R'" + run repeat_until_success_or_timeout 20 bash -c "docker exec ${CONTAINER_NAME} sh -c 'grep \"Subject: SPAM: \" /var/mail/localhost.localdomain/user1/.Junk/new/ -R'" assert_success } -@test "checking amavis: spam message is delivered to INBOX (MOVE_SPAM_TO_JUNK=0)" { - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container . mail_spam_moved_new) - - docker run -d --name mail_spam_moved_new \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_SPAMASSASSIN=1 \ - -e MOVE_SPAM_TO_JUNK=0 \ - -e PERMIT_DOCKER=container \ - -e SA_SPAM_SUBJECT="SPAM: " \ - -e SPAMASSASSIN_SPAM_TO_INBOX=1 \ - -h mail.my-domain.com -t "${NAME}" - - teardown() { docker rm -f mail_spam_moved_new; } - - wait_for_smtp_port_in_container mail_spam_moved_new +@test "${TEST_NAME_PREFIX} (Amavis) spam message delivered to INBOX" { + export CONTAINER_NAME=${CONTAINER2_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env MOVE_SPAM_TO_JUNK=0 + --env PERMIT_DOCKER=container + --env SA_SPAM_SUBJECT="SPAM: " + --env SPAMASSASSIN_SPAM_TO_INBOX=1 + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" # send a spam message - run docker exec mail_spam_moved_new /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt" + _run_in_container /bin/bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt" assert_success # message will be added to a queue with varying delay until amavis receives it - run repeat_until_success_or_timeout 60 sh -c "docker logs mail_spam_moved_new | grep 'Passed SPAM {RelayedTaggedInbound,Quarantined}'" + run repeat_until_success_or_timeout 60 bash -c "docker logs ${CONTAINER_NAME} | grep 'Passed SPAM {RelayedTaggedInbound,Quarantined}'" assert_success # spam moved to INBOX - run repeat_until_success_or_timeout 20 sh -c "docker exec mail_spam_moved_new sh -c 'grep \"Subject: SPAM: \" /var/mail/localhost.localdomain/user1/new/ -R'" + run repeat_until_success_or_timeout 20 bash -c "docker exec ${CONTAINER_NAME} sh -c 'grep \"Subject: SPAM: \" /var/mail/localhost.localdomain/user1/new/ -R'" assert_success } diff --git a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats index d5836dd3..7534e8d5 100644 --- a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats +++ b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats @@ -1,64 +1,61 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -function setup() { - local PRIVATE_CONFIG +TEST_NAME_PREFIX='Undefined spam subject:' - PRIVATE_CONFIG=$(duplicate_config_for_container .) - docker run -d --name mail_undef_spam_subject \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -e ENABLE_SPAMASSASSIN=1 \ - -e SA_SPAM_SUBJECT="undef" \ - --hostname mail.my-domain.com \ - --tty \ - "${NAME}" +CONTAINER1_NAME='dms-test_spam-undef-subject_1' +CONTAINER2_NAME='dms-test_spam-undef-subject_2' +CONTAINER_NAME=${CONTAINER2_NAME} - CONTAINER='mail_undef_spam_subject_2' - PRIVATE_CONFIG=$(duplicate_config_for_container . "${CONTAINER}") - docker run -d \ - -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ - -v "$(pwd)/test/onedir":/var/mail-state \ - -e ENABLE_CLAMAV=1 \ - -e SPOOF_PROTECTION=1 \ - -e ENABLE_SPAMASSASSIN=1 \ - -e REPORT_RECIPIENT=user1@localhost.localdomain \ - -e REPORT_SENDER=report1@mail.my-domain.com \ - -e SA_TAG=-5.0 \ - -e SA_TAG2=2.0 \ - -e SA_KILL=3.0 \ - -e SA_SPAM_SUBJECT="SPAM: " \ - -e VIRUSMAILS_DELETE_DELAY=7 \ - -e ENABLE_SRS=1 \ - -e ENABLE_MANAGESIEVE=1 \ - -e PERMIT_DOCKER=host \ - --name "${CONTAINER}" \ - --hostname mail.my-domain.com \ - --tty \ - --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" \ - "${NAME}" +function setup_file() { + local CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env SA_SPAM_SUBJECT='undef' + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - wait_for_finished_setup_in_container mail_undef_spam_subject - wait_for_finished_setup_in_container "${CONTAINER}" + # ulimit required for `ENABLE_SRS=1` + local CONTAINER_NAME=${CONTAINER2_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_CLAMAV=1 + --env SPOOF_PROTECTION=1 + --env ENABLE_SPAMASSASSIN=1 + --env REPORT_RECIPIENT=user1@localhost.localdomain + --env REPORT_SENDER=report1@mail.my-domain.com + --env SA_TAG=-5.0 + --env SA_TAG2=2.0 + --env SA_KILL=3.0 + --env SA_SPAM_SUBJECT="SPAM: " + --env VIRUSMAILS_DELETE_DELAY=7 + --env ENABLE_SRS=1 + --env ENABLE_MANAGESIEVE=1 + --env PERMIT_DOCKER=host + --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" + ) + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' } -function teardown() { - docker rm -f mail_undef_spam_subject "${CONTAINER}" +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } -@test "checking spamassassin: docker env variables are set correctly (custom)" { - run docker exec "${CONTAINER}" /bin/sh -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'" +@test "${TEST_NAME_PREFIX} Docker env variables are set correctly (custom)" { + _run_in_container bash -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'" assert_success - run docker exec "${CONTAINER}" /bin/sh -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" + _run_in_container bash -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" assert_success - run docker exec "${CONTAINER}" /bin/sh -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'" + _run_in_container bash -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'" assert_success - run docker exec "${CONTAINER}" /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'" + _run_in_container bash -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'" assert_success - run docker exec mail_undef_spam_subject /bin/sh -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" + run docker exec "${CONTAINER1_NAME}" bash -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" assert_success } diff --git a/test/tests/parallel/set2/template.bats b/test/tests/parallel/set2/template.bats index eb19cb58..7415b43a 100644 --- a/test/tests/parallel/set2/template.bats +++ b/test/tests/parallel/set2/template.bats @@ -6,7 +6,7 @@ load "${REPOSITORY_ROOT}/test/helper/common" # ? to identify the test easily TEST_NAME_PREFIX='template:' # ? must be unique -CONTAINER_NAME='dms-test-template' +CONTAINER_NAME='dms-test_template' # ? test setup diff --git a/test/tests/parallel/set2/tls/tls_cipherlists.bats b/test/tests/parallel/set2/tls/tls_cipherlists.bats index ffe6e643..745cc9ea 100644 --- a/test/tests/parallel/set2/tls/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls/tls_cipherlists.bats @@ -1,81 +1,128 @@ -#!/usr/bin/env bats -load "${REPOSITORY_ROOT}/test/test_helper/common" -# Globals ${BATS_TMPDIR} and ${NAME} -# `${NAME}` defaults to `mailserver-testing:ci` +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -function teardown() { - docker rm -f tls_test_cipherlists -} +TEST_NAME_PREFIX='[Security] TLS (cipher lists):' +CONTAINER_PREFIX='dms-test_tls-cipherlists' + +# NOTE: Tests cases here cannot be run concurrently: +# - The `testssl.txt` file configures `testssl.sh` to connect to `example.test` (TEST_DOMAIN) +# and this is set as a network alias to the DMS container being tested. +# - If multiple containers are active with this alias, the connection is not deterministic and will result +# in comparing the wrong results for a given variant. function setup_file() { - export DOMAIN="example.test" - export NETWORK="test-network" + export TEST_DOMAIN='example.test' + export TEST_FQDN="mail.${TEST_DOMAIN}" + export TEST_NETWORK='test-network' - # Shared config for TLS testing (read-only) + # Contains various certs for testing TLS support (read-only): export TLS_CONFIG_VOLUME - TLS_CONFIG_VOLUME="$(pwd)/test/test-files/ssl/${DOMAIN}/:/config/ssl/:ro" - # `${BATS_TMPDIR}` maps to `/tmp` - export TLS_RESULTS_DIR="${BATS_TMPDIR}/results" + TLS_CONFIG_VOLUME="${PWD}/test/test-files/ssl/${TEST_DOMAIN}/:/config/ssl/:ro" - # NOTE: If the network already exists, test will fail to start. - docker network create "${NETWORK}" - - # Copies all of `./test/config/` to specific directory for testing - # `${PRIVATE_CONFIG}` becomes `$(pwd)/test/duplicate_configs/` - export PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) + # Used for connecting testssl and DMS containers via network name `TEST_DOMAIN`: + # NOTE: If the network already exists, the test will fail to start + docker network create "${TEST_NETWORK}" # Pull `testssl.sh` image in advance to avoid it interfering with the `run` captured output. # Only interferes (potential test failure) with `assert_output` not `assert_success`? docker pull drwetter/testssl.sh:3.1dev + + # Only used in `should_support_expected_cipherlists()` to set a storage location for `testssl.sh` JSON output: + # `${BATS_TMPDIR}` maps to `/tmp`: https://bats-core.readthedocs.io/en/v1.8.2/writing-tests.html#special-variables + export TLS_RESULTS_DIR="${BATS_TMPDIR}/results" } function teardown_file() { - docker network rm "${NETWORK}" + docker network rm "${TEST_NETWORK}" } -@test "checking tls: cipher list - rsa intermediate" { - check_ports 'rsa' 'intermediate' +function teardown() { _default_teardown ; } + +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=intermediate' + RSA" { + configure_and_run_dms_container 'intermediate' 'rsa' + should_support_expected_cipherlists } -@test "checking tls: cipher list - rsa modern" { - check_ports 'rsa' 'modern' +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=intermediate' + ECDSA" { + configure_and_run_dms_container 'intermediate' 'ecdsa' + should_support_expected_cipherlists } -@test "checking tls: cipher list - ecdsa intermediate" { - check_ports 'ecdsa' 'intermediate' +# Only ECDSA with an RSA fallback is tested. +# There isn't a situation where RSA with an ECDSA fallback would make sense. +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=intermediate' + ECDSA with RSA fallback" { + configure_and_run_dms_container 'intermediate' 'ecdsa' 'rsa' + should_support_expected_cipherlists } -@test "checking tls: cipher list - ecdsa modern" { - check_ports 'ecdsa' 'modern' +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=modern' + RSA" { + configure_and_run_dms_container 'modern' 'rsa' + should_support_expected_cipherlists } - -# Only ECDSA with RSA fallback is tested. -# There isn't a situation where RSA with ECDSA fallback would make sense. -@test "checking tls: cipher list - ecdsa intermediate, with rsa fallback" { - check_ports 'ecdsa' 'intermediate' 'rsa' +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=modern' + ECDSA" { + configure_and_run_dms_container 'modern' 'ecdsa' + should_support_expected_cipherlists } -@test "checking tls: cipher list - ecdsa modern, with rsa fallback" { - check_ports 'ecdsa' 'modern' 'rsa' +@test "${TEST_NAME_PREFIX} 'TLS_LEVEL=modern' + ECDSA with RSA fallback" { + configure_and_run_dms_container 'modern' 'ecdsa' 'rsa' + should_support_expected_cipherlists } -function check_ports() { - local KEY_TYPE=$1 - local TLS_LEVEL=$2 +function configure_and_run_dms_container() { + local TLS_LEVEL=$1 + local KEY_TYPE=$2 local ALT_KEY_TYPE=$3 # Optional parameter - local KEY_TYPE_LABEL="${KEY_TYPE}" - # This is just to add a `_` delimiter between the two key types for readability + export TEST_VARIANT="${TLS_LEVEL}-${KEY_TYPE}" if [[ -n ${ALT_KEY_TYPE} ]] then - KEY_TYPE_LABEL="${KEY_TYPE}_${ALT_KEY_TYPE}" + TEST_VARIANT+="-${ALT_KEY_TYPE}" fi - local RESULTS_PATH="${KEY_TYPE_LABEL}/${TLS_LEVEL}" - collect_cipherlist_data + export CONTAINER_NAME="${CONTAINER_PREFIX}_${TEST_VARIANT}" + # The initial set of args is static across test cases: + local CUSTOM_SETUP_ARGUMENTS=( + --volume "${TLS_CONFIG_VOLUME}" + --network "${TEST_NETWORK}" + --network-alias "${TEST_DOMAIN}" + --env ENABLE_POP3=1 + --env SSL_TYPE="manual" + ) + # The remaining args are dependent upon test case vars: + CUSTOM_SETUP_ARGUMENTS+=( + --env TLS_LEVEL="${TLS_LEVEL}" + --env SSL_CERT_PATH="/config/ssl/cert.${KEY_TYPE}.pem" + --env SSL_KEY_PATH="/config/ssl/key.${KEY_TYPE}.pem" + ) + + if [[ -n ${ALT_KEY_TYPE} ]] + then + CUSTOM_SETUP_ARGUMENTS+=( + --env SSL_ALT_CERT_PATH="/config/ssl/cert.${ALT_KEY_TYPE}.pem" + --env SSL_ALT_KEY_PATH="/config/ssl/key.${ALT_KEY_TYPE}.pem" + ) + fi + + init_with_defaults + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_smtp_port_in_container "${CONTAINER_NAME}" +} + +function should_support_expected_cipherlists() { + # Make a directory with test user ownership. Avoids Docker creating this with root ownership. + # TODO: Can switch to filename prefix for JSON output when this is resolved: https://github.com/drwetter/testssl.sh/issues/1845 + local RESULTS_PATH="${TLS_RESULTS_DIR}/${TEST_VARIANT}" + mkdir -p "${RESULTS_PATH}" + + collect_cipherlists + verify_cipherlists +} + +# Verify that the collected results match our expected cipherlists: +function verify_cipherlists() { # SMTP: Opportunistic STARTTLS Explicit(25) # Needs to test against cipher lists specific to Port 25 ('_p25' parameter) check_cipherlists "${RESULTS_PATH}/port_25.json" '_p25' @@ -93,82 +140,56 @@ function check_ports() { check_cipherlists "${RESULTS_PATH}/port_995.json" } -function collect_cipherlist_data() { - local ALT_CERT=() - local ALT_KEY=() - - if [[ -n ${ALT_KEY_TYPE} ]] - then - ALT_CERT=(--env SSL_ALT_CERT_PATH="/config/ssl/cert.${ALT_KEY_TYPE}.pem") - ALT_KEY=(--env SSL_ALT_KEY_PATH="/config/ssl/key.${ALT_KEY_TYPE}.pem") - fi - - run docker run -d --name tls_test_cipherlists \ - --volume "${PRIVATE_CONFIG}/:/tmp/docker-mailserver/" \ - --volume "${TLS_CONFIG_VOLUME}" \ - --env ENABLE_POP3=1 \ - --env SSL_TYPE="manual" \ - --env SSL_CERT_PATH="/config/ssl/cert.${KEY_TYPE}.pem" \ - --env SSL_KEY_PATH="/config/ssl/key.${KEY_TYPE}.pem" \ - "${ALT_CERT[@]}" \ - "${ALT_KEY[@]}" \ - --env TLS_LEVEL="${TLS_LEVEL}" \ - --network "${NETWORK}" \ - --network-alias "${DOMAIN}" \ - --hostname "mail.${DOMAIN}" \ - --tty \ - "${NAME}" # Image name - - assert_success - - wait_for_tcp_port_in_container 25 tls_test_cipherlists +# Using `testssl.sh` we can test each port to collect a list of supported cipher suites (ordered): +function collect_cipherlists() { # NOTE: An rDNS query for the container IP will resolve to `..` - # Make directory with test user ownership. Avoids Docker creating with root ownership. - # TODO: Can switch to filename prefix for JSON output when this is resolved: https://github.com/drwetter/testssl.sh/issues/1845 - mkdir -p "${TLS_RESULTS_DIR}/${RESULTS_PATH}" - # For non-CI test runs, instead of removing prior test files after this test suite completes, # they're retained and overwritten by future test runs instead. Useful for inspection. # `--preference` reduces the test scope to the cipher suites reported as supported by the server. Completes in ~35% of the time. - local TESTSSL_CMD=(--quiet --file "/config/ssl/testssl.txt" --mode parallel --overwrite --preference) + local TESTSSL_CMD=( + --quiet + --file "/config/ssl/testssl.txt" + --mode parallel + --overwrite + --preference + ) # NOTE: Batch testing ports via `--file` doesn't properly bubble up failure. # If the failure for a test is misleading consider testing a single port with: - # local TESTSSL_CMD=(--quiet --jsonfile-pretty "${RESULTS_PATH}/port_${PORT}.json" --starttls smtp "${DOMAIN}:${PORT}") + # local TESTSSL_CMD=(--quiet --jsonfile-pretty "/output/port_${PORT}.json" --starttls smtp "${TEST_DOMAIN}:${PORT}") # TODO: Can use `jq` to check for failure when this is resolved: https://github.com/drwetter/testssl.sh/issues/1844 # `--user ":"` is a workaround: Avoids `permission denied` write errors for json output, uses `id` to match user uid & gid. run docker run --rm \ --user "$(id -u):$(id -g)" \ - --network "${NETWORK}" \ + --network "${TEST_NETWORK}" \ --volume "${TLS_CONFIG_VOLUME}" \ - --volume "${TLS_RESULTS_DIR}/${RESULTS_PATH}/:/output" \ + --volume "${RESULTS_PATH}:/output" \ --workdir "/output" \ drwetter/testssl.sh:3.1dev "${TESTSSL_CMD[@]}" assert_success } +# Compares the expected cipher lists against logged test results from `testssl.sh` +function check_cipherlists() { + local RESULTS_FILEPATH=$1 + local p25=$2 # optional suffix + + compare_cipherlist "cipherorder_TLSv1_2" "$(get_cipherlist "TLSv1_2${p25}")" + compare_cipherlist "cipherorder_TLSv1_3" "$(get_cipherlist 'TLSv1_3')" +} + # Use `jq` to extract a specific cipher list from the target`testssl.sh` results json output file function compare_cipherlist() { local TARGET_CIPHERLIST=$1 - local RESULTS_FILE=$2 - local EXPECTED_CIPHERLIST=$3 + local EXPECTED_CIPHERLIST=$2 - run jq '.scanResult[0].serverPreferences[] | select(.id=="'"${TARGET_CIPHERLIST}"'") | .finding' "${TLS_RESULTS_DIR}/${RESULTS_FILE}" + run jq '.scanResult[0].serverPreferences[] | select(.id=="'"${TARGET_CIPHERLIST}"'") | .finding' "${RESULTS_FILEPATH}" assert_success assert_output "${EXPECTED_CIPHERLIST}" } -# Compares the expected cipher lists against logged test results from `testssl.sh` -function check_cipherlists() { - local RESULTS_FILE=$1 - local p25=$2 # optional suffix - - compare_cipherlist "cipherorder_TLSv1_2" "${RESULTS_FILE}" "$(get_cipherlist "TLSv1_2${p25}")" - compare_cipherlist "cipherorder_TLSv1_3" "${RESULTS_FILE}" "$(get_cipherlist 'TLSv1_3')" -} - # Expected cipher lists. Should match `TLS_LEVEL` cipher lists set in `scripts/helpers/ssl.sh`. # Excluding Port 25 which uses defaults from Postfix after applying `smtpd_tls_exclude_ciphers` rules. # NOTE: If a test fails, look at the `check_ports` params, then update the corresponding associative key's value @@ -185,34 +206,33 @@ function get_cipherlist() { # Associative array for easy querying of required cipher list declare -A CIPHER_LIST - CIPHER_LIST["rsa_intermediate_TLSv1_2"]='"ECDHE-RSA-CHACHA20-POLY1305 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256"' - CIPHER_LIST["rsa_modern_TLSv1_2"]='"ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384"' + # RSA: + CIPHER_LIST["intermediate-rsa_TLSv1_2"]='"ECDHE-RSA-CHACHA20-POLY1305 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256"' + CIPHER_LIST["modern-rsa_TLSv1_2"]='"ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384"' # ECDSA: - CIPHER_LIST["ecdsa_intermediate_TLSv1_2"]='"ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384"' - CIPHER_LIST["ecdsa_modern_TLSv1_2"]='"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305"' + CIPHER_LIST["intermediate-ecdsa_TLSv1_2"]='"ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA256 ECDHE-ECDSA-AES256-SHA384"' + CIPHER_LIST["modern-ecdsa_TLSv1_2"]='"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305"' # ECDSA + RSA fallback, dual cert support: - CIPHER_LIST["ecdsa_rsa_intermediate_TLSv1_2"]='"ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA256 ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 ECDHE-ECDSA-AES256-SHA384 DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256"' - CIPHER_LIST["ecdsa_rsa_modern_TLSv1_2"]='"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384"' + CIPHER_LIST["intermediate-ecdsa-rsa_TLSv1_2"]='"ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-AES128-SHA256 ECDHE-RSA-AES128-SHA256 ECDHE-RSA-AES256-SHA384 ECDHE-ECDSA-AES256-SHA384 DHE-RSA-AES128-SHA256 DHE-RSA-AES256-SHA256"' + CIPHER_LIST["modern-ecdsa-rsa_TLSv1_2"]='"ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES256-GCM-SHA384"' - # Port 25 - # TLSv1_2 has different server order and also includes ARIA, CCM, DHE+CHACHA20-POLY1305 cipher suites: - CIPHER_LIST["rsa_intermediate_TLSv1_2_p25"]='"ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES256-CCM8 DHE-RSA-AES256-CCM ECDHE-ARIA256-GCM-SHA384 DHE-RSA-ARIA256-GCM-SHA384 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES256-SHA256 ARIA256-GCM-SHA384 ECDHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-CCM8 DHE-RSA-AES128-CCM ECDHE-ARIA128-GCM-SHA256 DHE-RSA-ARIA128-GCM-SHA256 ECDHE-RSA-AES128-SHA256 DHE-RSA-AES128-SHA256 ARIA128-GCM-SHA256"' - # Port 25 is unaffected by `TLS_LEVEL` profiles, it has the same TLS v1.2 cipher list under both: - CIPHER_LIST["rsa_modern_TLSv1_2_p25"]=${CIPHER_LIST["rsa_intermediate_TLSv1_2_p25"]} - + # Port 25 has a different server order, and also includes ARIA, CCM, DHE+CHACHA20-POLY1305 cipher suites: + # RSA (Port 25): + CIPHER_LIST["intermediate-rsa_TLSv1_2_p25"]='"ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-GCM-SHA384 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-CHACHA20-POLY1305 DHE-RSA-AES256-CCM8 DHE-RSA-AES256-CCM ECDHE-ARIA256-GCM-SHA384 DHE-RSA-ARIA256-GCM-SHA384 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES256-SHA256 ARIA256-GCM-SHA384 ECDHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-CCM8 DHE-RSA-AES128-CCM ECDHE-ARIA128-GCM-SHA256 DHE-RSA-ARIA128-GCM-SHA256 ECDHE-RSA-AES128-SHA256 DHE-RSA-AES128-SHA256 ARIA128-GCM-SHA256"' # ECDSA (Port 25): - CIPHER_LIST["ecdsa_intermediate_TLSv1_2_p25"]='"ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES256-CCM8 ECDHE-ECDSA-AES256-CCM ECDHE-ECDSA-ARIA256-GCM-SHA384 ECDHE-ECDSA-AES256-SHA384 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES128-CCM8 ECDHE-ECDSA-AES128-CCM ECDHE-ECDSA-ARIA128-GCM-SHA256 ECDHE-ECDSA-AES128-SHA256"' - CIPHER_LIST["ecdsa_modern_TLSv1_2_p25"]=${CIPHER_LIST["ecdsa_intermediate_TLSv1_2_p25"]} - + CIPHER_LIST["intermediate-ecdsa_TLSv1_2_p25"]='"ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES256-CCM8 ECDHE-ECDSA-AES256-CCM ECDHE-ECDSA-ARIA256-GCM-SHA384 ECDHE-ECDSA-AES256-SHA384 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES128-CCM8 ECDHE-ECDSA-AES128-CCM ECDHE-ECDSA-ARIA128-GCM-SHA256 ECDHE-ECDSA-AES128-SHA256"' # ECDSA + RSA fallback, dual cert support (Port 25): - CIPHER_LIST["ecdsa_rsa_intermediate_TLSv1_2_p25"]='"ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES256-CCM8 ECDHE-ECDSA-AES256-CCM DHE-RSA-AES256-CCM8 DHE-RSA-AES256-CCM ECDHE-ECDSA-ARIA256-GCM-SHA384 ECDHE-ARIA256-GCM-SHA384 DHE-RSA-ARIA256-GCM-SHA384 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES256-SHA256 ARIA256-GCM-SHA384 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES128-CCM8 ECDHE-ECDSA-AES128-CCM DHE-RSA-AES128-CCM8 DHE-RSA-AES128-CCM ECDHE-ECDSA-ARIA128-GCM-SHA256 ECDHE-ARIA128-GCM-SHA256 DHE-RSA-ARIA128-GCM-SHA256 ECDHE-ECDSA-AES128-SHA256 ECDHE-RSA-AES128-SHA256 DHE-RSA-AES128-SHA256 ARIA128-GCM-SHA256"' - CIPHER_LIST["ecdsa_rsa_modern_TLSv1_2_p25"]=${CIPHER_LIST["ecdsa_rsa_intermediate_TLSv1_2_p25"]} + CIPHER_LIST["intermediate-ecdsa-rsa_TLSv1_2_p25"]='"ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES256-GCM-SHA384 DHE-RSA-AES256-GCM-SHA384 ECDHE-ECDSA-CHACHA20-POLY1305 ECDHE-RSA-CHACHA20-POLY1305 DHE-RSA-CHACHA20-POLY1305 ECDHE-ECDSA-AES256-CCM8 ECDHE-ECDSA-AES256-CCM DHE-RSA-AES256-CCM8 DHE-RSA-AES256-CCM ECDHE-ECDSA-ARIA256-GCM-SHA384 ECDHE-ARIA256-GCM-SHA384 DHE-RSA-ARIA256-GCM-SHA384 ECDHE-ECDSA-AES256-SHA384 ECDHE-RSA-AES256-SHA384 DHE-RSA-AES256-SHA256 ARIA256-GCM-SHA384 ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-RSA-AES128-GCM-SHA256 DHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES128-CCM8 ECDHE-ECDSA-AES128-CCM DHE-RSA-AES128-CCM8 DHE-RSA-AES128-CCM ECDHE-ECDSA-ARIA128-GCM-SHA256 ECDHE-ARIA128-GCM-SHA256 DHE-RSA-ARIA128-GCM-SHA256 ECDHE-ECDSA-AES128-SHA256 ECDHE-RSA-AES128-SHA256 DHE-RSA-AES128-SHA256 ARIA128-GCM-SHA256"' + # Port 25 is unaffected by `TLS_LEVEL` profiles, thus no difference for modern: + CIPHER_LIST["modern-rsa_TLSv1_2_p25"]=${CIPHER_LIST["intermediate-rsa_TLSv1_2_p25"]} + CIPHER_LIST["modern-ecdsa_TLSv1_2_p25"]=${CIPHER_LIST["intermediate-ecdsa_TLSv1_2_p25"]} + CIPHER_LIST["modern-ecdsa-rsa_TLSv1_2_p25"]=${CIPHER_LIST["intermediate-ecdsa-rsa_TLSv1_2_p25"]} - local TARGET_QUERY="${KEY_TYPE_LABEL}_${TLS_LEVEL}_${TLS_VERSION}" + local TARGET_QUERY="${TEST_VARIANT}_${TLS_VERSION}" echo "${CIPHER_LIST[${TARGET_QUERY}]}" fi } diff --git a/test/tests/parallel/set2/tls/tls_letsencrypt.bats b/test/tests/parallel/set2/tls/tls_letsencrypt.bats index e764527b..fc6b649e 100644 --- a/test/tests/parallel/set2/tls/tls_letsencrypt.bats +++ b/test/tests/parallel/set2/tls/tls_letsencrypt.bats @@ -1,29 +1,22 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" -load "${REPOSITORY_ROOT}/test/test_helper/tls" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/tls" -# Globals referenced from `test_helper/common`: -# TEST_NAME TEST_FQDN TEST_TMP_CONFIG +TEST_NAME_PREFIX='[Security] TLS (SSL_TYPE=letsencrypt):' +CONTAINER1_NAME='dms-test_tls-letsencrypt_default-hostname' +CONTAINER2_NAME='dms-test_tls-letsencrypt_fallback-domainname' +CONTAINER3_NAME='dms-test_tls-letsencrypt_support-acme-json' +export TEST_FQDN='mail.example.test' -# Requires maintenance (TODO): Yes -# Can run tests in parallel?: No +function teardown() { _default_teardown ; } -# Not parallelize friendly when TEST_NAME is static, -# presently name of test file: `mail_ssl_letsencrypt`. -# -# Also shares a common TEST_TMP_CONFIG local folder, -# Instead of individual PRIVATE_CONFIG copies. -# For this test that is a non-issue, unless run in parallel. - - -# Applies to all tests: -function setup_file() { +# Similar to BATS `setup()` method, but invoked manually after +# CONTAINER_NAME has been adjusted for the running testcase. +function _initial_setup() { init_with_defaults - # Override default to match the hostname we want to test against instead: - export TEST_FQDN='mail.example.test' - # Prepare certificates in the letsencrypt supported file structure: - # Note Certbot uses `privkey.pem`. + # NOTE: Certbot uses `privkey.pem`. # `fullchain.pem` is currently what's detected, but we're actually providing the equivalent of `cert.pem` here. # TODO: Verify format/structure is supported for nginx-proxy + acme-companion (uses `acme.sh` to provision). @@ -36,45 +29,38 @@ function setup_file() { _copy_to_letsencrypt_storage 'example.test/with_ca/ecdsa/key.rsa.pem' 'example.test/privkey.pem' } -# Not used -# function teardown_file() { -# } - -function teardown() { - docker rm -f "${TEST_NAME}" -} - # Should detect and choose the cert for FQDN `mail.example.test` (HOSTNAME): -@test "ssl(letsencrypt): Should default to HOSTNAME (mail.example.test)" { - local TARGET_DOMAIN='mail.example.test' +@test "${TEST_NAME_PREFIX} Should default to HOSTNAME (${TEST_FQDN})" { + export CONTAINER_NAME=${CONTAINER1_NAME} + _initial_setup - local TEST_DOCKER_ARGS=( + local TARGET_DOMAIN=${TEST_FQDN} + local CUSTOM_SETUP_ARGUMENTS=( --volume "${TEST_TMP_CONFIG}/letsencrypt/${TARGET_DOMAIN}/:/etc/letsencrypt/live/${TARGET_DOMAIN}/:ro" --env PERMIT_DOCKER='container' --env SSL_TYPE='letsencrypt' ) + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - common_container_setup 'TEST_DOCKER_ARGS' - - #test hostname has certificate files + # Test that certificate files exist for the configured `hostname`: _should_have_valid_config "${TARGET_DOMAIN}" 'privkey.pem' 'fullchain.pem' _should_succesfully_negotiate_tls "${TARGET_DOMAIN}" _should_not_support_fqdn_in_cert 'example.test' } - # Should detect and choose cert for FQDN `example.test` (DOMAINNAME), # as fallback when no cert for FQDN `mail.example.test` (HOSTNAME) exists: -@test "ssl(letsencrypt): Should fallback to DOMAINNAME (example.test)" { - local TARGET_DOMAIN='example.test' +@test "${TEST_NAME_PREFIX} Should fallback to DOMAINNAME (example.test)" { + export CONTAINER_NAME=${CONTAINER2_NAME} + _initial_setup - local TEST_DOCKER_ARGS=( + local TARGET_DOMAIN='example.test' + local CUSTOM_SETUP_ARGUMENTS=( --volume "${TEST_TMP_CONFIG}/letsencrypt/${TARGET_DOMAIN}/:/etc/letsencrypt/live/${TARGET_DOMAIN}/:ro" --env PERMIT_DOCKER='container' --env SSL_TYPE='letsencrypt' ) - - common_container_setup 'TEST_DOCKER_ARGS' + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' #test domain has certificate files _should_have_valid_config "${TARGET_DOMAIN}" 'privkey.pem' 'fullchain.pem' @@ -87,34 +73,36 @@ function teardown() { # # NOTE: Currently all of the `acme.json` configs have the FQDN match a SAN value, # all Subject CN (`main` in acme.json) are `Smallstep Leaf` which is not an FQDN. -# While valid for that field, it does mean there is no test coverage against `main`. -@test "ssl(letsencrypt): Traefik 'acme.json' (*.example.test)" { - # This test group changes to certs signed with an RSA Root CA key, - # These certs all support both FQDNs: `mail.example.test` and `example.test`, - # Except for the wildcard cert `*.example.test`, which intentionally excluded `example.test` when created. - # We want to maintain the same FQDN (mail.example.test) between the _acme_ecdsa and _acme_rsa tests. - local LOCAL_BASE_PATH="${PWD}/test/test-files/ssl/example.test/with_ca/rsa" +# While not using a FQDN is valid for that field, +# it does mean there is no test coverage against the `acme.json` field `main`. +@test "${TEST_NAME_PREFIX} Traefik 'acme.json' (*.example.test)" { + export CONTAINER_NAME=${CONTAINER3_NAME} + _initial_setup - # Change default Root CA cert used for verifying chain of trust with openssl: + # Override the `_initial_setup()` default Root CA cert (used for verifying the chain of trust via `openssl`): # shellcheck disable=SC2034 local TEST_CA_CERT="${TEST_FILES_CONTAINER_PATH}/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem" + # This test group switches to certs that are signed with an RSA Root CA key instead. + # All of these certs support both FQDNs (`mail.example.test` and `example.test`), + # Except for the wildcard cert (`*.example.test`), that was created with `example.test` intentionally excluded from SAN. + # We want to maintain the same FQDN (`mail.example.test`) between the _acme_ecdsa and _acme_rsa tests. + local LOCAL_BASE_PATH="${PWD}/test/test-files/ssl/example.test/with_ca/rsa" + function _prepare() { # Default `acme.json` for _acme_ecdsa test: cp "${LOCAL_BASE_PATH}/ecdsa.acme.json" "${TEST_TMP_CONFIG}/letsencrypt/acme.json" # TODO: Provision wildcard certs via Traefik to inspect if `example.test` non-wildcard is also added to the cert. - # shellcheck disable=SC2034 - local TEST_DOCKER_ARGS=( + local CUSTOM_SETUP_ARGUMENTS=( --volume "${TEST_TMP_CONFIG}/letsencrypt/acme.json:/etc/letsencrypt/acme.json:ro" --env LOG_LEVEL='trace' --env PERMIT_DOCKER='container' --env SSL_DOMAIN='*.example.test' --env SSL_TYPE='letsencrypt' ) - - common_container_setup 'TEST_DOCKER_ARGS' - wait_for_service "${TEST_NAME}" 'changedetector' + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + wait_for_service "${CONTAINER_NAME}" 'changedetector' # Wait until the changedetector service startup delay is over: repeat_until_success_or_timeout 20 sh -c "$(_get_service_logs 'changedetector') | grep 'Changedetector is ready'" @@ -184,7 +172,6 @@ function teardown() { # Test Methods # - # Check that Dovecot and Postfix are configured to use a cert for the expected FQDN: function _should_have_valid_config() { local EXPECTED_FQDN=${1} @@ -199,16 +186,14 @@ function _should_have_valid_config() { # CMD ${1} run in container with output checked to match value of ${2}: function _has_matching_line() { - run docker exec "${TEST_NAME}" sh -c "${1} | grep '${2}'" + _run_in_container bash -c "${1} | grep '${2}'" assert_output "${2}" } - # # Traefik `acme.json` specific # - # It should log success of extraction for the expected domain and restart Postfix. function _should_have_succeeded_at_extraction() { local EXPECTED_DOMAIN=${1} @@ -249,7 +234,7 @@ function _should_have_service_reload_count() { local NUM_RELOADS=${1} # Count how many times processes (like Postfix and Dovecot) have been reloaded by the `changedetector` service: - run docker exec "${TEST_NAME}" sh -c "grep -c 'Completed handling of detected change' /var/log/supervisor/changedetector.log" + _run_in_container grep --count 'Completed handling of detected change' '/var/log/supervisor/changedetector.log' assert_output "${NUM_RELOADS}" } @@ -265,12 +250,10 @@ function _should_have_expected_files() { _should_be_equal_in_content "${LE_CERT_PATH}" "${EXPECTED_CERT_PATH}" } - # # Misc # - # Rename test certificate files to match the expected file structure for letsencrypt: function _copy_to_letsencrypt_storage() { local SRC=${1} @@ -280,14 +263,18 @@ function _copy_to_letsencrypt_storage() { FQDN_DIR=$(echo "${DEST}" | cut -d '/' -f1) mkdir -p "${TEST_TMP_CONFIG}/letsencrypt/${FQDN_DIR}" - cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}" + if ! cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}" + then + echo "Could not copy cert file '${SRC}'' to '${DEST}'" >&2 + exit 1 + fi } function _should_be_equal_in_content() { local CONTAINER_PATH=${1} local LOCAL_PATH=${2} - run docker exec "${TEST_NAME}" sh -c "cat ${CONTAINER_PATH}" + _run_in_container /bin/bash -c "cat ${CONTAINER_PATH}" assert_output "$(cat "${LOCAL_PATH}")" assert_success } @@ -295,13 +282,13 @@ function _should_be_equal_in_content() { function _get_service_logs() { local SERVICE=${1:-'mailserver'} - local CMD_LOGS=(docker exec "${TEST_NAME}" "supervisorctl tail -2200 ${SERVICE}") + local CMD_LOGS=(docker exec "${CONTAINER_NAME}" "supervisorctl tail -2200 ${SERVICE}") # As the `mailserver` service logs are not stored in a file but output to stdout/stderr, # The `supervisorctl tail` command won't work; we must instead query via `docker logs`: if [[ ${SERVICE} == 'mailserver' ]] then - CMD_LOGS=(docker logs "${TEST_NAME}") + CMD_LOGS=(docker logs "${CONTAINER_NAME}") fi echo "${CMD_LOGS[@]}" diff --git a/test/tests/parallel/set2/tls/tls_manual.bats b/test/tests/parallel/set2/tls/tls_manual.bats index d2c34cc5..bf0d2d86 100644 --- a/test/tests/parallel/set2/tls/tls_manual.bats +++ b/test/tests/parallel/set2/tls/tls_manual.bats @@ -1,8 +1,11 @@ -#!/usr/bin/env bats -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +TEST_NAME_PREFIX='[Security] TLS (SSL_TYPE=manual):' +CONTAINER_NAME='dms-test_tls-manual' function setup_file() { - # Internal copies made by `start-mailserver.sh`: + # Internal copies made by `scripts/helpers/ssl.sh`: export PRIMARY_KEY='/etc/dms/tls/key' export PRIMARY_CERT='/etc/dms/tls/cert' export FALLBACK_KEY='/etc/dms/tls/fallback_key' @@ -14,98 +17,101 @@ function setup_file() { export SSL_ALT_KEY_PATH='/config/ssl/key.rsa.pem' export SSL_ALT_CERT_PATH='/config/ssl/cert.rsa.pem' - local PRIVATE_CONFIG - export DOMAIN_SSL_MANUAL='example.test' - PRIVATE_CONFIG=$(duplicate_config_for_container .) + export TEST_DOMAIN='example.test' - docker run -d --name mail_manual_ssl \ - --volume "${PRIVATE_CONFIG}/:/tmp/docker-mailserver/" \ - --volume "$(pwd)/test/test-files/ssl/${DOMAIN_SSL_MANUAL}/with_ca/ecdsa/:/config/ssl/:ro" \ - --env LOG_LEVEL='trace' \ - --env SSL_TYPE='manual' \ - --env TLS_LEVEL='modern' \ - --env SSL_KEY_PATH="${SSL_KEY_PATH}" \ - --env SSL_CERT_PATH="${SSL_CERT_PATH}" \ - --env SSL_ALT_KEY_PATH="${SSL_ALT_KEY_PATH}" \ - --env SSL_ALT_CERT_PATH="${SSL_ALT_CERT_PATH}" \ - --hostname "mail.${DOMAIN_SSL_MANUAL}" \ - --tty \ - "${NAME}" # Image name + local CUSTOM_SETUP_ARGUMENTS=( + --volume "${PWD}/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/:/config/ssl/:ro" + --env LOG_LEVEL='trace' + --env SSL_TYPE='manual' + --env TLS_LEVEL='modern' + --env SSL_KEY_PATH="${SSL_KEY_PATH}" + --env SSL_CERT_PATH="${SSL_CERT_PATH}" + --env SSL_ALT_KEY_PATH="${SSL_ALT_KEY_PATH}" + --env SSL_ALT_CERT_PATH="${SSL_ALT_CERT_PATH}" + ) - wait_for_finished_setup_in_container mail_manual_ssl + init_with_defaults + # Override the default set in `common_container_setup`: + export TEST_FQDN="mail.${TEST_DOMAIN}" + common_container_setup 'CUSTOM_SETUP_ARGUMENTS' } -function teardown_file() { - docker rm -f mail_manual_ssl +function teardown_file() { _default_teardown ; } + +@test "${TEST_NAME_PREFIX} ENV vars provided are valid files" { + _run_in_container [ -f "${SSL_CERT_PATH}" ] + assert_success + + _run_in_container [ -f "${SSL_KEY_PATH}" ] + assert_success + + _run_in_container [ -f "${SSL_ALT_CERT_PATH}" ] + assert_success + + _run_in_container [ -f "${SSL_ALT_KEY_PATH}" ] + assert_success } -@test "checking ssl: ENV vars provided are valid files" { - assert docker exec mail_manual_ssl [ -f "${SSL_CERT_PATH}" ] - assert docker exec mail_manual_ssl [ -f "${SSL_KEY_PATH}" ] - assert docker exec mail_manual_ssl [ -f "${SSL_ALT_CERT_PATH}" ] - assert docker exec mail_manual_ssl [ -f "${SSL_ALT_KEY_PATH}" ] -} - -@test "checking ssl: manual configuration is correct" { +@test "${TEST_NAME_PREFIX} manual configuration is correct" { local DOVECOT_CONFIG_SSL='/etc/dovecot/conf.d/10-ssl.conf' - run docker exec mail_manual_ssl grep '^smtpd_tls_chain_files =' '/etc/postfix/main.cf' + _run_in_container grep '^smtpd_tls_chain_files =' '/etc/postfix/main.cf' assert_success assert_output "smtpd_tls_chain_files = ${PRIMARY_KEY} ${PRIMARY_CERT} ${FALLBACK_KEY} ${FALLBACK_CERT}" - run docker exec mail_manual_ssl grep '^ssl_key =' "${DOVECOT_CONFIG_SSL}" + _run_in_container grep '^ssl_key =' "${DOVECOT_CONFIG_SSL}" assert_success assert_output "ssl_key = <${PRIMARY_KEY}" - run docker exec mail_manual_ssl grep '^ssl_cert =' "${DOVECOT_CONFIG_SSL}" + _run_in_container grep '^ssl_cert =' "${DOVECOT_CONFIG_SSL}" assert_success assert_output "ssl_cert = <${PRIMARY_CERT}" - run docker exec mail_manual_ssl grep '^ssl_alt_key =' "${DOVECOT_CONFIG_SSL}" + _run_in_container grep '^ssl_alt_key =' "${DOVECOT_CONFIG_SSL}" assert_success assert_output "ssl_alt_key = <${FALLBACK_KEY}" - run docker exec mail_manual_ssl grep '^ssl_alt_cert =' "${DOVECOT_CONFIG_SSL}" + _run_in_container grep '^ssl_alt_cert =' "${DOVECOT_CONFIG_SSL}" assert_success assert_output "ssl_alt_cert = <${FALLBACK_CERT}" } -@test "checking ssl: manual configuration copied files correctly " { - run docker exec mail_manual_ssl cmp -s "${PRIMARY_KEY}" "${SSL_KEY_PATH}" +@test "${TEST_NAME_PREFIX} manual configuration copied files correctly " { + _run_in_container cmp -s "${PRIMARY_KEY}" "${SSL_KEY_PATH}" assert_success - run docker exec mail_manual_ssl cmp -s "${PRIMARY_CERT}" "${SSL_CERT_PATH}" + _run_in_container cmp -s "${PRIMARY_CERT}" "${SSL_CERT_PATH}" assert_success # Fallback cert - run docker exec mail_manual_ssl cmp -s "${FALLBACK_KEY}" "${SSL_ALT_KEY_PATH}" + _run_in_container cmp -s "${FALLBACK_KEY}" "${SSL_ALT_KEY_PATH}" assert_success - run docker exec mail_manual_ssl cmp -s "${FALLBACK_CERT}" "${SSL_ALT_CERT_PATH}" + _run_in_container cmp -s "${FALLBACK_CERT}" "${SSL_ALT_CERT_PATH}" assert_success } -@test "checking ssl: manual cert works correctly" { - wait_for_tcp_port_in_container 587 mail_manual_ssl +@test "${TEST_NAME_PREFIX} manual cert works correctly" { + wait_for_tcp_port_in_container 587 "${CONTAINER_NAME}" local TEST_COMMAND=(timeout 1 openssl s_client -connect mail.example.test:587 -starttls smtp) local RESULT # Should fail as a chain of trust is required to verify successfully: - RESULT=$(docker exec mail_manual_ssl "${TEST_COMMAND[@]}" | grep 'Verification error:') + RESULT=$(docker exec "${CONTAINER_NAME}" "${TEST_COMMAND[@]}" | grep 'Verification error:') assert_equal "${RESULT}" 'Verification error: unable to verify the first certificate' # Provide the Root CA cert for successful verification: local CA_CERT='/config/ssl/ca-cert.ecdsa.pem' - assert docker exec mail_manual_ssl [ -f "${CA_CERT}" ] - RESULT=$(docker exec mail_manual_ssl "${TEST_COMMAND[@]}" -CAfile "${CA_CERT}" | grep 'Verification: OK') + assert docker exec "${CONTAINER_NAME}" [ -f "${CA_CERT}" ] + RESULT=$(docker exec "${CONTAINER_NAME}" "${TEST_COMMAND[@]}" -CAfile "${CA_CERT}" | grep 'Verification: OK') assert_equal "${RESULT}" 'Verification: OK' } -@test "checking ssl: manual cert changes are picked up by check-for-changes" { +@test "${TEST_NAME_PREFIX} manual cert changes are picked up by check-for-changes" { printf '%s' 'someThingsChangedHere' \ - >>"$(pwd)/test/test-files/ssl/${DOMAIN_SSL_MANUAL}/with_ca/ecdsa/key.ecdsa.pem" + >>"$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" - run timeout 15 docker exec mail_manual_ssl bash -c "tail -F /var/log/supervisor/changedetector.log | sed '/Manual certificates have changed/ q'" + run timeout 15 docker exec "${CONTAINER_NAME}" bash -c "tail -F /var/log/supervisor/changedetector.log | sed '/Manual certificates have changed/ q'" assert_success - sed -i '/someThingsChangedHere/d' "$(pwd)/test/test-files/ssl/${DOMAIN_SSL_MANUAL}/with_ca/ecdsa/key.ecdsa.pem" + sed -i '/someThingsChangedHere/d' "$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" } diff --git a/test/tests/parallel/set3/dovecot_inet_protocol.bats b/test/tests/parallel/set3/dovecot_inet_protocol.bats index b6fdf3cb..ce7bd832 100644 --- a/test/tests/parallel/set3/dovecot_inet_protocol.bats +++ b/test/tests/parallel/set3/dovecot_inet_protocol.bats @@ -2,9 +2,14 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" TEST_NAME_PREFIX='Dovecot protocols:' +CONTAINER1_NAME='dms-test_dovecot_protocols_all' +CONTAINER2_NAME='dms-test_dovecot_protocols_ipv4' +CONTAINER3_NAME='dms-test_dovecot_protocols_ipv6' + +function teardown() { _default_teardown ; } @test "${TEST_NAME_PREFIX} dual-stack IP configuration" { - local CONTAINER_NAME='dms-test-dovecot_protocols_all' + export CONTAINER_NAME=${CONTAINER1_NAME} local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=) init_with_defaults @@ -13,12 +18,10 @@ TEST_NAME_PREFIX='Dovecot protocols:' _run_in_container grep '^#listen = \*, ::' /etc/dovecot/dovecot.conf assert_success assert_output '#listen = *, ::' - - docker rm -f "${CONTAINER_NAME}" } @test "${TEST_NAME_PREFIX} IPv4 configuration" { - local CONTAINER_NAME='dms-test-dovecot_protocols_ipv4' + export CONTAINER_NAME=${CONTAINER2_NAME} local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=ipv4) init_with_defaults @@ -27,12 +30,10 @@ TEST_NAME_PREFIX='Dovecot protocols:' _run_in_container grep '^listen = \*$' /etc/dovecot/dovecot.conf assert_success assert_output 'listen = *' - - docker rm -f "${CONTAINER_NAME}" } @test "${TEST_NAME_PREFIX} IPv6 configuration" { - local CONTAINER_NAME='dms-test-dovecot_protocols_ipv6' + export CONTAINER_NAME=${CONTAINER3_NAME} local CUSTOM_SETUP_ARGUMENTS=(--env DOVECOT_INET_PROTOCOLS=ipv6) init_with_defaults @@ -41,6 +42,4 @@ TEST_NAME_PREFIX='Dovecot protocols:' _run_in_container grep '^listen = \[::\]$' /etc/dovecot/dovecot.conf assert_success assert_output 'listen = [::]' - - docker rm -f "${CONTAINER_NAME}" } diff --git a/test/tests/parallel/set3/helper-functions.bats b/test/tests/parallel/set3/helper-functions.bats index 24ee2f68..510d79c7 100644 --- a/test/tests/parallel/set3/helper-functions.bats +++ b/test/tests/parallel/set3/helper-functions.bats @@ -2,7 +2,7 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" TEST_NAME_PREFIX='helper functions inside container:' -CONTAINER_NAME='dms-test-helper_functions' +CONTAINER_NAME='dms-test_helper_functions' function setup_file() { init_with_defaults From 0bbec095296f72b74d99ea0b759b2ff8c90fb819 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 3 Jan 2023 19:11:36 +1300 Subject: [PATCH 4/5] refactor: Parallel Tests - `disabled_clamav_spamassassin`: - Just shuffling the test order around, and removing the restart test at the end which doesn't make sense. - `postscreen`: - Now uses common helper for getting container IP - Does not appear to need the `NET_ADMIN` capability? - Reduced startup time for the 2nd container + additional context about it's relevance. - Test cases are largely the same, but refactored the `nc` alternative that properly waits it's turn. This only needs to run once. Added additional commentary and made into a generic method if needed in other tests. - `fail2ban`: - Use the common container IP helper method. - Postscreen isn't affecting this test, it's not required to do the much slower exchange with the mail server when sending a login failure. - IP being passed into ENV is no longer necessary. - `sleep 5` in the related test cases doesn't seem necessary, can better rely on polling with timeout. - `sleep 10` for `setup.sh` also doesn't appear to be necessary. - `postgrey`: - Reduced POSTGREY_DELAY to 3, which shaves a fair amount of wasted time while still verifying the delay works. - One of the checks in `main.cf` doesn't seem to need to know about the earlier spamhaus portion of the line to work, removed. - Better test case descriptions. - Improved log matching via standard method that better documents the expected triplet under test. - Removed a redundant whitelist file and test that didn't seem to have any relevance. Added a TODO with additional notes about a concern with these tests. - Reduced test time as 8 second timeouts from `-w 8` don't appear to be required, better to poll with grep instead. - Replaced `wc -l` commands with a new method to assert expected line count, better enabling assertions on the actual output. - `undef_spam_subject`: - Split to two separate test cases, and initialize each container in their case instead of `setup_file()`, allowing for using the default `teardown()` method (and slight benefit if running in parallel). - `permit_docker`: - Not a parallel test, but I realized that the repeat helper methods don't necessarily play well with `run` as the command (can cause false positive of what was successful). --- test/helper/common.bash | 8 ++ .../nc_templates/postgrey_whitelist_local.txt | 9 -- .../disabled_clamav_spamassassin.bats | 17 +-- .../parallel/set1/spam_virus/fail2ban.bats | 43 ++---- .../set1/spam_virus/postgrey_enabled.bats | 124 ++++++++++++------ .../parallel/set1/spam_virus/postscreen.bats | 66 +++++++--- .../set1/spam_virus/undef_spam_subject.bats | 26 ++-- test/tests/serial/permit_docker.bats | 3 +- 8 files changed, 175 insertions(+), 121 deletions(-) delete mode 100644 test/test-files/nc_templates/postgrey_whitelist_local.txt diff --git a/test/helper/common.bash b/test/helper/common.bash index 2eba1086..26c66137 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -10,6 +10,8 @@ __load_bats_helper # ------------------------------------------------------------------- # like _run_in_container_explicit but infers ${1} by using the ENV CONTAINER_NAME +# WARNING: Careful using this with _until_success_or_timeout methods, +# which can be misleading in the success of `run`, not the command given to `run`. function _run_in_container() { run docker exec "${CONTAINER_NAME}" "${@}" } @@ -36,6 +38,12 @@ function check_if_process_is_running() { docker exec "${CONTAINER_NAME}" pgrep "${PROGRAM_NAME}" } +# @param ${1} target container name [IF UNSET: ${CONTAINER_NAME}] +function get_container_ip() { + local TARGET_CONTAINER_NAME=${1:-${CONTAINER_NAME}} + docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${TARGET_CONTAINER_NAME}" +} + # ------------------------------------------------------------------- # @param ${1} timeout diff --git a/test/test-files/nc_templates/postgrey_whitelist_local.txt b/test/test-files/nc_templates/postgrey_whitelist_local.txt deleted file mode 100644 index 0358d5ca..00000000 --- a/test/test-files/nc_templates/postgrey_whitelist_local.txt +++ /dev/null @@ -1,9 +0,0 @@ -request=smtpd_access_policy -protocol_state=RCPT -protocol_name=ESMTP -client_address=127.0.0.1 -client_name=whitelistlocal.tld -helo_name=whitelistlocal.tld -sender=test@whitelistlocal.tld -recipient=user1@localhost.localdomain - diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 23fd00c0..b90d9e4b 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -29,22 +29,17 @@ function teardown_file() { _default_teardown ; } assert_failure } -@test "${TEST_NAME_PREFIX} SA - Amavis integration should not be active" { - _run_in_container /bin/sh -c "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'" - assert_success -} - @test "${TEST_NAME_PREFIX} ClamAV - Amavis integration should not be active" { _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log assert_failure } -@test "${TEST_NAME_PREFIX} SA should not be called" { +@test "${TEST_NAME_PREFIX} SA - Amavis integration should not be active" { + _run_in_container bash -c "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'" + assert_success +} + +@test "${TEST_NAME_PREFIX} SA - should not have been called" { _run_in_container grep -i 'connect to /var/run/clamav/clamd.ctl failed' /var/log/mail/mail.log assert_failure } - -@test "${TEST_NAME_PREFIX} ClamAV process should not be restarted when killed" { - _run_in_container /bin/bash -c "pkill -f clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" - assert_failure -} diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 29a5e620..a065e06e 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -5,10 +5,6 @@ TEST_NAME_PREFIX='Fail2Ban:' CONTAINER1_NAME='dms-test_fail2ban' CONTAINER2_NAME='dms-test_fail2ban_fail-auth-mailer' -function get_container2_ip() { - docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${CONTAINER2_NAME}" -} - function setup_file() { export CONTAINER_NAME @@ -17,6 +13,7 @@ function setup_file() { --env ENABLE_FAIL2BAN=1 --env POSTSCREEN_ACTION=ignore --cap-add=NET_ADMIN + # NOTE: May no longer be needed with newer F2B: --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" ) init_with_defaults @@ -25,7 +22,6 @@ function setup_file() { # Create a container which will send wrong authentications and should get banned CONTAINER_NAME=${CONTAINER2_NAME} - local CUSTOM_SETUP_ARGUMENTS=(--env MAIL_FAIL2BAN_IP="$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' ${CONTAINER1_NAME})") init_with_defaults common_container_setup 'CUSTOM_SETUP_ARGUMENTS' @@ -76,30 +72,21 @@ function teardown_file() { done } -# NOTE: This test case is fragile if other test cases were to be run concurrently +# NOTE: This test case is fragile if other test cases were to be run concurrently. +# - After multiple login fails and a slight delay, f2b will ban that IP. +# - You could hard-code `sleep 5` on both cases to avoid the alternative assertions, +# but the polling + piping into grep approach here reliably minimizes the delay. @test "${TEST_NAME_PREFIX} ban ip on multiple failed login" { - # can't pipe the file as usual due to postscreen - # respecting postscreen_greet_wait time and talking in turn): - - # shellcheck disable=SC1004 - for _ in {1,2} - do - docker exec "${CONTAINER2_NAME}" /bin/bash -c \ - 'exec 3<>/dev/tcp/${MAIL_FAIL2BAN_IP}/25 && \ - while IFS= read -r cmd; do \ - head -1 <&3; \ - [[ ${cmd} == "EHLO"* ]] && sleep 6; \ - echo ${cmd} >&3; \ - done < "/tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt"' - done - - sleep 5 + CONTAINER1_IP=$(get_container_ip ${CONTAINER1_NAME}) + # Trigger a ban by failing to login twice: + _run_in_container_explicit "${CONTAINER2_NAME}" bash -c "nc ${CONTAINER1_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt" + _run_in_container_explicit "${CONTAINER2_NAME}" bash -c "nc ${CONTAINER1_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt" # Checking that CONTAINER2_IP is banned in "${CONTAINER1_NAME}" - CONTAINER2_IP=$(get_container2_ip) - _run_in_container fail2ban-client status postfix-sasl + CONTAINER2_IP=$(get_container_ip ${CONTAINER2_NAME}) + run repeat_in_container_until_success_or_timeout 10 "${CONTAINER_NAME}" bash -c "fail2ban-client status postfix-sasl | grep -F '${CONTAINER2_IP}'" assert_success - assert_output --partial "${CONTAINER2_IP}" + assert_output --partial 'Banned IP list:' # Checking that CONTAINER2_IP is banned by nftables _run_in_container bash -c 'nft list set inet f2b-table addr-set-postfix-sasl' @@ -107,11 +94,11 @@ function teardown_file() { assert_output --partial "elements = { ${CONTAINER2_IP} }" } +# NOTE: Depends on previous test case, if no IP was banned at this point, it passes regardless.. @test "${TEST_NAME_PREFIX} unban ip works" { - CONTAINER2_IP=$(get_container2_ip) + CONTAINER2_IP=$(get_container_ip ${CONTAINER2_NAME}) _run_in_container fail2ban-client set postfix-sasl unbanip "${CONTAINER2_IP}" assert_success - sleep 5 # Checking that CONTAINER2_IP is unbanned in "${CONTAINER1_NAME}" _run_in_container fail2ban-client status postfix-sasl @@ -189,8 +176,6 @@ function teardown_file() { _run_in_container fail2ban-client set dovecot banip 192.0.66.4 _run_in_container fail2ban-client set dovecot banip 192.0.66.5 - sleep 10 - # Originally: run ./setup.sh -c "${CONTAINER1_NAME}" fail2ban _run_in_container setup fail2ban assert_output --regexp '^Banned in dovecot:.*192\.0\.66\.4' diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 4244d7d5..bc57c73a 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -10,7 +10,7 @@ function setup_file() { --env ENABLE_POSTGREY=1 --env PERMIT_DOCKER=container --env POSTGREY_AUTO_WHITELIST_CLIENTS=5 - --env POSTGREY_DELAY=15 + --env POSTGREY_DELAY=3 --env POSTGREY_MAX_AGE=35 --env POSTGREY_TEXT="Delayed by Postgrey" ) @@ -24,20 +24,20 @@ function setup_file() { function teardown_file() { _default_teardown ; } -@test "${TEST_NAME_PREFIX} /etc/postfix/main.cf correctly edited" { - _run_in_container bash -c "grep -F 'zen.spamhaus.org=127.0.0.[2..11], check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf | wc -l" +@test "${TEST_NAME_PREFIX} should have added Postgrey to 'main.cf:check_policy_service'" { + _run_in_container grep -F 'check_policy_service inet:127.0.0.1:10023' /etc/postfix/main.cf assert_success - assert_output 1 + _should_output_number_of_lines 1 } -@test "${TEST_NAME_PREFIX} /etc/default/postgrey correctly edited and has the default values" { - _run_in_container bash -c "grep '^POSTGREY_OPTS=\"--inet=127.0.0.1:10023 --delay=15 --max-age=35 --auto-whitelist-clients=5\"$' /etc/default/postgrey | wc -l" +@test "${TEST_NAME_PREFIX} should have configured /etc/default/postgrey with default values and ENV overrides" { + _run_in_container grep -F 'POSTGREY_OPTS="--inet=127.0.0.1:10023 --delay=3 --max-age=35 --auto-whitelist-clients=5"' /etc/default/postgrey assert_success - assert_output 1 + _should_output_number_of_lines 1 - _run_in_container bash -c "grep '^POSTGREY_TEXT=\"Delayed by Postgrey\"$' /etc/default/postgrey | wc -l" + _run_in_container grep -F 'POSTGREY_TEXT="Delayed by Postgrey"' /etc/default/postgrey assert_success - assert_output 1 + _should_output_number_of_lines 1 } @test "${TEST_NAME_PREFIX} Postgrey is running" { @@ -45,48 +45,96 @@ function teardown_file() { _default_teardown ; } assert_success } -@test "${TEST_NAME_PREFIX} there should be a log entry about a new greylisted e-mail user@external.tld in /var/log/mail/mail.log" { - #editing the postfix config in order to ensure that postgrey handles the test e-mail. The other spam checks at smtpd_recipient_restrictions would interfere with it. +@test "${TEST_NAME_PREFIX} should initially reject (greylist) mail from 'user@external.tld'" { + # Modify the postfix config in order to ensure that postgrey handles the test e-mail. + # The other spam checks in `main.cf:smtpd_recipient_restrictions` would interfere with testing postgrey. _run_in_container bash -c "sed -ie 's/permit_sasl_authenticated.*policyd-spf,$//g' /etc/postfix/main.cf" _run_in_container bash -c "sed -ie 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' /etc/postfix/main.cf" _run_in_container bash -c "sed -ie 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' /etc/postfix/main.cf" _run_in_container bash -c "sed -ie 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' /etc/postfix/main.cf" _run_in_container postfix reload - _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" - sleep 5 #ensure that the information has been written into the log - _run_in_container bash -c "grep -i 'action=greylist.*user@external\.tld' /var/log/mail/mail.log | wc -l" - assert_success - assert_output 1 + # Send test mail (it should fail to deliver): + _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' + + # Confirm mail was greylisted: + _should_have_log_entry \ + 'action=greylist' \ + 'reason=new' \ + 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' + + repeat_until_success_or_timeout 10 _run_in_container grep \ + 'Recipient address rejected: Delayed by Postgrey' \ + /var/log/mail/mail.log } -@test "${TEST_NAME_PREFIX} there should be a log entry about the retried and passed e-mail user@external.tld in /var/log/mail/mail.log" { - sleep 20 #wait 20 seconds so that postgrey would accept the message - _run_in_container bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/postgrey.txt" - sleep 8 +# NOTE: This test case depends on the previous one +@test "${TEST_NAME_PREFIX} should accept mail from 'user@external.tld' after POSTGREY_DELAY duration" { + # Wait until `$POSTGREY_DELAY` seconds pass before trying again: + sleep 3 + # Retry delivering test mail (it should be trusted this time): + _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' - _run_in_container bash -c "grep -i 'action=pass, reason=triplet found.*user@external\.tld' /var/log/mail/mail.log | wc -l" - assert_success - assert_output 1 + # Confirm postgrey permitted delivery (triplet is now trusted): + _should_have_log_entry \ + 'action=pass' \ + 'reason=triplet found' \ + 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' } -@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" { - _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt" - _run_in_container bash -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" - assert_success - assert_output 1 + +# NOTE: These two whitelist tests use `test-files/nc_templates/` instead of `test-files/email-templates`. +# - This allows to bypass the SMTP protocol on port 25, and send data directly to Postgrey instead. +# - Appears to be a workaround due to `client_name=localhost` when sent from Postfix. +# - Could send over port 25 if whitelisting `localhost`, +# - However this does not help verify that the actual client HELO address is properly whitelisted? +# - It'd also cause the earlier greylist test to fail. +# - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround: +@test "${TEST_NAME_PREFIX} should whitelist sender 'user@whitelist.tld'" { + _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt' '10023' + + _should_have_log_entry \ + 'action=pass' \ + 'reason=client whitelist' \ + 'client_address=127.0.0.1/32, sender=test@whitelist.tld, recipient=user1@localhost.localdomain' } -@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted local and passed e-mail user@whitelistlocal.tld in /var/log/mail/mail.log" { - _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_local.txt" - _run_in_container bash -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l" - assert_success - assert_output 1 +@test "${TEST_NAME_PREFIX} should whitelist recipient 'user2@otherdomain.tld'" { + _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt' '10023' + + _should_have_log_entry \ + 'action=pass' \ + 'reason=recipient whitelist' \ + 'client_address=127.0.0.1/32, sender=test@nonwhitelist.tld, recipient=user2@otherdomain.tld' } -@test "${TEST_NAME_PREFIX} there should be a log entry about the whitelisted recipient user2@otherdomain.tld in /var/log/mail/mail.log" { - _run_in_container bash -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt" - _run_in_container bash -c "grep -i 'action=pass, reason=recipient whitelist' /var/log/mail/mail.log | wc -l" - assert_success - assert_output 1 +function _send_test_mail() { + local MAIL_TEMPLATE=$1 + local PORT=${2:-25} + + # `-w 0` terminates the connection after sending the template, it does not wait for a response. + # This is required for port 10023, otherwise the connection never drops. + # It could increase the number of seconds to wait for port 25 to allow for asserting a response, + # but that would enforce the delay in tests for port 10023. + _run_in_container bash -c "nc -w 0 0.0.0.0 ${PORT} < ${MAIL_TEMPLATE}" +} + +function _should_have_log_entry() { + local ACTION=$1 + local REASON=$2 + local TRIPLET=$3 + + # Allow some extra time for logs to update to avoids a false-positive failure: + run_until_success_or_timeout 10 docker exec "${CONTAINER_NAME}" grep \ + "${ACTION}, ${REASON}," \ + /var/log/mail/mail.log + + # Log entry matched should be for the expected triplet: + assert_output --partial "${TRIPLET}" + _should_output_number_of_lines 1 +} + +# `lines` is a special BATS variable updated via `run`: +function _should_output_number_of_lines() { + assert_equal "${#lines[@]}" $1 } diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index e44c5bb1..492bb96b 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -6,49 +6,77 @@ CONTAINER1_NAME='dms-test_postscreen_enforce' CONTAINER2_NAME='dms-test_postscreen_sender' function setup() { - MAIL_POSTSCREEN_IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' "${CONTAINER1_NAME}") + CONTAINER1_IP=$(get_container_ip ${CONTAINER1_NAME}) } function setup_file() { - local CONTAINER_NAME=${CONTAINER1_NAME} + export CONTAINER_NAME + + CONTAINER_NAME=${CONTAINER1_NAME} local CUSTOM_SETUP_ARGUMENTS=( --env POSTSCREEN_ACTION=enforce - --cap-add=NET_ADMIN ) init_with_defaults common_container_setup 'CUSTOM_SETUP_ARGUMENTS' wait_for_smtp_port_in_container "${CONTAINER_NAME}" - local CONTAINER_NAME=${CONTAINER2_NAME} + # A standard DMS instance to send mail from: + # NOTE: None of DMS is actually used for this (just bash + nc). + CONTAINER_NAME=${CONTAINER2_NAME} init_with_defaults - common_container_setup - wait_for_smtp_port_in_container "${CONTAINER_NAME}" + # No need to wait for DMS to be ready for this container: + common_container_create + run docker start "${CONTAINER_NAME}" + assert_success + + # Set default implicit container fallback for helpers: + CONTAINER_NAME=${CONTAINER1_NAME} } function teardown_file() { docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } -@test "${TEST_NAME_PREFIX} talk too fast" { - run docker exec "${CONTAINER2_NAME}" /bin/sh -c "nc ${MAIL_POSTSCREEN_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt" +@test "${TEST_NAME_PREFIX} should fail login when talking out of turn" { + _run_in_container_explicit "${CONTAINER2_NAME}" bash -c "nc ${CONTAINER1_IP} 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login.txt" assert_success + assert_output --partial '502 5.5.2 Error: command not recognized' - repeat_until_success_or_timeout 10 run docker exec "${CONTAINER1_NAME}" grep 'COMMAND PIPELINING' /var/log/mail/mail.log - assert_success + # Expected postscreen log entry: + _run_in_container cat /var/log/mail/mail.log + assert_output --partial 'COMMAND PIPELINING' } -@test "${TEST_NAME_PREFIX} positive test (respecting postscreen_greet_wait time and talking in turn)" { - for _ in {1,2}; do - # shellcheck disable=SC1004 - docker exec "${CONTAINER2_NAME}" /bin/bash -c \ - 'exec 3<>/dev/tcp/'"${MAIL_POSTSCREEN_IP}"'/25 && \ +@test "${TEST_NAME_PREFIX} should successfully login (respecting postscreen_greet_wait time)" { + # NOTE: Sometimes fails on first attempt (trying too soon?), + # Instead of a `run` + asserting partial, Using repeat + internal grep match: + repeat_until_success_or_timeout 10 _should_wait_turn_speaking_smtp \ + "${CONTAINER2_NAME}" \ + "${CONTAINER1_IP}" \ + '/tmp/docker-mailserver-test/auth/smtp-auth-login.txt' \ + 'Authentication successful' + + # Expected postscreen log entry: + _run_in_container cat /var/log/mail/mail.log + assert_output --partial 'PASS NEW' +} + +# When postscreen is active, it prevents the usual method of piping a file through nc: +# (Won't work: _run_in_container_explicit "${CLIENT_CONTAINER_NAME}" bash -c "nc ${TARGET_CONTAINER_IP} 25 < ${SMTP_TEMPLATE}") +# The below workaround respects `postscreen_greet_wait` time (default 6 sec), talking to the mail-server in turn: +# https://www.postfix.org/postconf.5.html#postscreen_greet_wait +function _should_wait_turn_speaking_smtp() { + local CLIENT_CONTAINER_NAME=$1 + local TARGET_CONTAINER_IP=$2 + local SMTP_TEMPLATE=$3 + local EXPECTED=$4 + + local UGLY_WORKAROUND='exec 3<>/dev/tcp/'"${TARGET_CONTAINER_IP}"'/25 && \ while IFS= read -r cmd; do \ head -1 <&3; \ [[ ${cmd} == "EHLO"* ]] && sleep 6; \ echo ${cmd} >&3; \ - done < "/tmp/docker-mailserver-test/auth/smtp-auth-login.txt"' - done + done < '"${SMTP_TEMPLATE}" - repeat_until_success_or_timeout 10 run docker exec "${CONTAINER1_NAME}" grep 'PASS NEW ' /var/log/mail/mail.log - assert_success + docker exec "${CLIENT_CONTAINER_NAME}" bash -c "${UGLY_WORKAROUND}" | grep "${EXPECTED}" } diff --git a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats index 7534e8d5..b3b16bbe 100644 --- a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats +++ b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats @@ -7,8 +7,10 @@ CONTAINER1_NAME='dms-test_spam-undef-subject_1' CONTAINER2_NAME='dms-test_spam-undef-subject_2' CONTAINER_NAME=${CONTAINER2_NAME} -function setup_file() { - local CONTAINER_NAME=${CONTAINER1_NAME} +function teardown() { _default_teardown ; } + +@test "${TEST_NAME_PREFIX} 'SA_SPAM_SUBJECT=undef' should update Amavis config" { + export CONTAINER_NAME=${CONTAINER1_NAME} local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 @@ -17,8 +19,14 @@ function setup_file() { init_with_defaults common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - # ulimit required for `ENABLE_SRS=1` - local CONTAINER_NAME=${CONTAINER2_NAME} + _run_in_container bash -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" + assert_success +} + +# TODO: Unclear why some of these ENV are relevant for the test? +@test "${TEST_NAME_PREFIX} Docker env variables are set correctly (custom)" { + export CONTAINER_NAME=${CONTAINER2_NAME} + local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_CLAMAV=1 --env SPOOF_PROTECTION=1 @@ -33,17 +41,12 @@ function setup_file() { --env ENABLE_SRS=1 --env ENABLE_MANAGESIEVE=1 --env PERMIT_DOCKER=host + # NOTE: ulimit required for `ENABLE_SRS=1` until running a newer `postsrsd` --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" ) init_with_defaults common_container_setup 'CUSTOM_SETUP_ARGUMENTS' -} -function teardown_file() { - docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" -} - -@test "${TEST_NAME_PREFIX} Docker env variables are set correctly (custom)" { _run_in_container bash -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'" assert_success @@ -55,7 +58,4 @@ function teardown_file() { _run_in_container bash -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'" assert_success - - run docker exec "${CONTAINER1_NAME}" bash -c "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" - assert_success } diff --git a/test/tests/serial/permit_docker.bats b/test/tests/serial/permit_docker.bats index fb18d072..2e56e297 100644 --- a/test/tests/serial/permit_docker.bats +++ b/test/tests/serial/permit_docker.bats @@ -71,8 +71,7 @@ teardown_file() { # we should be able to send from the other container on the second network! run docker exec mail_smtponly_second_network_sender /bin/sh -c "nc mail_smtponly_second_network 25 < /tmp/docker-mailserver-test/email-templates/smtp-only.txt" assert_output --partial "250 2.0.0 Ok: queued as " - repeat_until_success_or_timeout 60 run docker exec mail_smtponly_second_network /bin/sh -c 'grep -cE "to=.*status\=sent" /var/log/mail/mail.log' - [[ ${status} -ge 0 ]] + repeat_in_container_until_success_or_timeout 60 mail_smtponly_second_network /bin/sh -c 'grep -cE "to=.*status\=sent" /var/log/mail/mail.log' } @test "checking PERMIT_DOCKER: none" { From 52987e32e7584fcc188aa0eff39971f271bf008d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 3 Jan 2023 21:05:06 +1300 Subject: [PATCH 5/5] docs: Revise contributing advice for tests --- docs/content/contributing/general.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md index d6bdcb94..c3d681a6 100644 --- a/docs/content/contributing/general.md +++ b/docs/content/contributing/general.md @@ -15,7 +15,7 @@ When refactoring, writing or altering scripts or other files, adhere to these ru To run the test suite, you will need to -1. [Install Docker] +1. [Install Docker][get-docker] 2. Install `jq` (under Ubuntu, use `sudo apt-get -y install jq`) 3. Execute `git submodule update --init --recursive` if you haven't already initialized the git submodules 4. Execute `make clean all` @@ -32,15 +32,20 @@ To run the test suite, you will need to !!! warning "Test Output when Running in Parallel" - When running test in parallel, BATS will run more than one test at any given time. This can result in output not being exactly what you'd expect, i.e. there could be _more_ or _less_ than you'd think. Those writing tests need to take care of this. Always test with `make clean generate-accounts tests/parallel/setX`. + [When running tests in parallel][docs-bats-parallel] (_with `make clean generate-accounts tests/parallel/setX`_), BATS will delay outputting the results until completing all test cases within a file. -??? tip "Running a Specific Test" + This also delays test failures as a result. When troubleshooting parallel set tests, you may prefer to run them serially as advised below. - To run a specific test, use `make build generate-accounts test/`, where `` is the file name of the test (_for more precision use a relative path: `test/test/`_) **excluding** the `.bats` suffix. + When writing tests, ensure that parallel set tests still pass when run in parallel. You need to account for other tests running in parallel that may interfere with your own tests logic. - Example: To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`). +??? tip "Run a Specific Test" -[Install Docker]: https://docs.docker.com/get-docker/ + Run `make build generate-accounts test/`, where `` is the file name of the test **excluding** the `.bats` suffix (_use a relative path if needing to be more specific: `test//`_). + + Multiple test files can be run sequentially with a `,` delimiter between file names: + `make test/tls_letsencrypt,tls_manual` + + **Example:** To run only the tests in `template.bats`, use `make test/template` (_or with relative path: `make test/parallel/set2/template`_). ## Documentation @@ -51,3 +56,6 @@ docker run --rm -it -p 8000:8000 -v "${PWD}:/docs" squidfunk/mkdocs-material ``` This serves the documentation on your local machine on port `8000`. Each change will be hot-reloaded onto the page you view, just edit, save and look at the result. + +[get-docker]: https://docs.docker.com/get-docker/ +[docs-bats-parallel]: https://bats-core.readthedocs.io/en/v1.8.2/usage.html#parallel-execution