ci: misc test enhancements (#2815)

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

.github/workflows/generic_test.yml

@@ -48,4 +48,3 @@ jobs:
         run: make generate-accounts tests
         env:
           CI: true
-          NAME: mailserver-testing:ci

Makefile

@@ -1,8 +1,7 @@
 SHELL = /bin/bash
 
-NAME        ?= mailserver-testing:ci
-VCS_REVISION = $(shell git rev-parse --short HEAD)
-VCS_VERSION  = $(shell cat VERSION)
+export NAME       ?= mailserver-testing:ci
+export IMAGE_NAME := $(NAME)
 
 # -----------------------------------------------
 # --- Generic Build Targets ---------------------
@@ -12,8 +11,8 @@ all: lint build backup generate-accounts tests clean
 
 build:
 	@ DOCKER_BUILDKIT=1 docker build --tag $(NAME) \
-		--build-arg VCS_VERSION=$(VCS_VERSION) \
-		--build-arg VCS_REVISION=$(VCS_REVISION) \
+		--build-arg VCS_VERSION=$(shell git rev-parse --short HEAD) \
+		--build-arg VCS_REVISION=$(shell cat VERSION) \
 		.
 
 backup:
@@ -43,11 +42,10 @@ generate-accounts:
 	@ docker run --rm -e MASTER_USER=masterusername -e MASTER_PASS=masterpassword -t $(NAME) /bin/sh -c 'echo "$$MASTER_USER|$$(doveadm pw -s SHA512-CRYPT -u $$MASTER_USER -p $$MASTER_PASS)"' > test/config/dovecot-masters.cf
 
 tests:
-	@ NAME=$(NAME) ./test/bats/bin/bats --timing test/*.bats
+	@ ./test/bats/bin/bats --timing test/*.bats
 
-.PHONY: ALWAYS_RUN
-test/%.bats: ALWAYS_RUN
-	@ ./test/bats/bin/bats $@
+test/%:
+	@ ./test/bats/bin/bats --timing $@.bats
 
 lint: eclint hadolint shellcheck
 

test/clamav.bats

@@ -0,0 +1,79 @@
+load 'test_helper/common'
+
+TEST_NAME_PREFIX='ClamAV:'
+CONTAINER_NAME='dms-test-clamav'
+RUN_COMMAND=('run' 'docker' 'exec' "${CONTAINER_NAME}")
+
+function setup_file() {
+  local PRIVATE_CONFIG
+  PRIVATE_CONFIG=$(duplicate_config_for_container . "${CONTAINER_NAME}")
+
+  docker run --rm --detach --tty \
+    --name "${CONTAINER_NAME}" \
+    --hostname mail.my-domain.com \
+    --volume "${PRIVATE_CONFIG}:/tmp/docker-mailserver" \
+    --volume "${PWD}/test/test-files:/tmp/docker-mailserver-test:ro" \
+    --env ENABLE_AMAVIS=1 \
+    --env AMAVIS_LOGLEVEL=2 \
+    --env ENABLE_CLAMAV=1 \
+    --env ENABLE_UPDATE_CHECK=0 \
+    --env ENABLE_SPAMASSASSIN=0 \
+    --env ENABLE_FAIL2BAN=0 \
+    --env PERMIT_DOCKER=host \
+    --env CLAMAV_MESSAGE_SIZE_LIMIT=30M \
+    --env LOG_LEVEL=debug \
+    "${IMAGE_NAME}"
+
+  wait_for_finished_setup_in_container "${CONTAINER_NAME}"
+
+  # wait for ClamAV to be fully setup or we will get errors on the log
+  repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl
+
+  wait_for_service "${CONTAINER_NAME}" postfix
+  wait_for_smtp_port_in_container "${CONTAINER_NAME}"
+
+  "${RUN_COMMAND[@]}" bash -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt"
+  assert_success
+
+  wait_for_empty_mail_queue_in_container "${CONTAINER_NAME}"
+}
+
+function teardown_file() {
+  docker rm -f "${CONTAINER_NAME}"
+}
+
+@test "${TEST_NAME_PREFIX} process clamd is running" {
+  "${RUN_COMMAND[@]}" bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
+  assert_success
+}
+
+@test "${TEST_NAME_PREFIX} log files exist at /var/log/mail directory" {
+  "${RUN_COMMAND[@]}" bash -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log'| wc -l"
+  assert_success
+  assert_output 3
+}
+
+@test "${TEST_NAME_PREFIX} should be identified by Amavis" {
+  "${RUN_COMMAND[@]}" grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
+  assert_success
+}
+
+@test "${TEST_NAME_PREFIX} freshclam cron is enabled" {
+  "${RUN_COMMAND[@]}" bash -c "grep '/usr/bin/freshclam' -r /etc/cron.d"
+  assert_success
+}
+
+@test "${TEST_NAME_PREFIX} env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" {
+  "${RUN_COMMAND[@]}" grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf
+  assert_success
+}
+
+@test "${TEST_NAME_PREFIX} rejects virus" {
+  "${RUN_COMMAND[@]}" bash -c "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep '<virus@external.tld> -> <user1@localhost.localdomain>'"
+  assert_success
+}
+
+@test "${TEST_NAME_PREFIX} process clamd restarts when killed" {
+  "${RUN_COMMAND[@]}" bash -c "pkill clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
+  assert_success
+}

test/setup-cli.bats

@@ -86,6 +86,9 @@ function teardown_file() {
   refute_output --partial 'Password must not be empty'
   assert_success
 
+  # NOTE: this was put in place for the next test `setup.sh email del` to properly work.
+  wait_until_change_detection_event_completes "${TEST_NAME}"
+
   # `postfix-accounts.cf` should have an updated password hash stored:
   local NEW_PASS_HASH
   NEW_PASS_HASH=$(grep "${MAIL_ACCOUNT}" "${DATABASE_ACCOUNTS}" | awk -F '|' '{print $2}')
@@ -111,14 +114,11 @@ function teardown_file() {
   run ./setup.sh -c "${TEST_NAME}" email del -y "${MAIL_ACCOUNT}"
   assert_success
 
-  # Mail storage for account was actually removed by `-y`:
   # NOTE: Sometimes the directory still exists, possibly from change detection
-  # of the previous test (`email udpate`) triggering. If this does not fix it,
-  # use `wait_until_change_detection_event_completes()` in that test to ensure
-  # consistency, or create another user to delete.
+  # of the previous test (`email udpate`) triggering. Therefore, the function
+  # `wait_until_change_detection_event_completes was added to the
+  # `setup.sh email update` test.
   repeat_in_container_until_success_or_timeout 60 "${TEST_NAME}" bash -c '[[ ! -d /var/mail/example.com/user ]]'
-  #run docker exec "${TEST_NAME}" bash -c '[[ ! -d /var/mail/example.com/user ]]'
-  #assert_success
 
   # Account is not present in `postfix-accounts.cf`:
   run grep "${MAIL_ACCOUNT}" "${TEST_TMP_CONFIG}/postfix-accounts.cf"

test/test_helper/common.bash

@@ -6,8 +6,8 @@ load 'test_helper/bats-assert/load'
 NAME=${NAME:-mailserver-testing:ci}
 
 # default timeout is 120 seconds
-TEST_TIMEOUT_IN_SECONDS=${TEST_TIMEOUT_IN_SECONDS-120}
-NUMBER_OF_LOG_LINES=${NUMBER_OF_LOG_LINES-10}
+TEST_TIMEOUT_IN_SECONDS=${TEST_TIMEOUT_IN_SECONDS:-120}
+NUMBER_OF_LOG_LINES=${NUMBER_OF_LOG_LINES:-10}
 
 # @param ${1} timeout
 # @param --fatal-test <command eval string> additional test whose failure aborts immediately

test/tests.bats

@@ -1,8 +1,5 @@
 load 'test_helper/common'
 
-export IMAGE_NAME
-IMAGE_NAME="${NAME}"
-
 setup_file() {
   local PRIVATE_CONFIG
   PRIVATE_CONFIG=$(duplicate_config_for_container . mail)
@@ -15,14 +12,13 @@ setup_file() {
     -v "$(pwd)/test/onedir":/var/mail-state \
     -e AMAVIS_LOGLEVEL=2 \
     -e CLAMAV_MESSAGE_SIZE_LIMIT=30M \
-    -e ENABLE_CLAMAV=1 \
+    -e ENABLE_CLAMAV=0 \
     -e ENABLE_MANAGESIEVE=1 \
     -e ENABLE_QUOTAS=1 \
     -e ENABLE_SPAMASSASSIN=1 \
     -e ENABLE_SRS=1 \
     -e ENABLE_UPDATE_CHECK=0 \
     -e LOG_LEVEL='debug' \
-    -e PERMIT_DOCKER=container \
     -e PERMIT_DOCKER=host \
     -e PFLOGSUMM_TRIGGER=logrotate \
     -e REPORT_RECIPIENT=user1@localhost.localdomain \
@@ -52,17 +48,13 @@ setup_file() {
 
   # this relies on the checksum file being updated after all changes have been applied
   wait_until_change_detection_event_completes mail
-
-  # wait for ClamAV to be fully setup or we will get errors on the log
-  repeat_in_container_until_success_or_timeout 60 mail test -e /var/run/clamav/clamd.ctl
-
+  wait_for_service mail postfix
   wait_for_smtp_port_in_container mail
 
   # The first mail sent leverages an assert for better error output if a failure occurs:
   run docker exec mail /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-spam.txt"
   assert_success
 
-  docker exec mail /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt"
   docker exec mail /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-alias-external.txt"
   docker exec mail /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-alias-local.txt"
   docker exec mail /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-alias-recipient-delimiter.txt"
@@ -128,9 +120,9 @@ teardown_file() {
   assert_success
 }
 
-@test "checking process: clamd" {
+@test "checking process: clamd (is not runnning)" {
   run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
-  assert_success
+  assert_failure
 }
 
 @test "checking process: new" {
@@ -201,9 +193,8 @@ teardown_file() {
 #
 
 @test "checking logs: mail related logs should be located in a subdirectory" {
-  run docker exec mail /bin/sh -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log'|wc -l"
+  run docker exec mail /bin/sh -c "ls -1 /var/log/mail/ | grep -E 'mail.log'"
   assert_success
-  assert_output 3
 }
 
 #
@@ -332,12 +323,6 @@ EOF
   assert_output 1
 }
 
-@test "checking smtp: rejects virus" {
-  run docker exec mail /bin/sh -c "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep external.tld=virus@my-domain.com | wc -l"
-  assert_success
-  assert_output 1
-}
-
 @test "checking smtp: not advertising smtputf8" {
   # Dovecot does not support SMTPUTF8, so while we can send we cannot receive
   # Better disable SMTPUTF8 support entirely if we can't handle it correctly
@@ -436,21 +421,6 @@ EOF
   assert_success
 }
 
-
-#
-# ClamAV
-#
-
-@test "checking ClamAV: should be listed in amavis when enabled" {
-  run docker exec mail grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log
-  assert_success
-}
-
-@test "checking ClamAV: CLAMAV_MESSAGE_SIZE_LIMIT" {
-  run docker exec mail grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf
-  assert_success
-}
-
 #
 # postsrsd
 #
@@ -481,9 +451,9 @@ EOF
 # system
 #
 
-@test "checking system: freshclam cron is enabled" {
+@test "checking system: freshclam cron is disabled" {
   run docker exec mail bash -c "grep '/usr/bin/freshclam' -r /etc/cron.d"
-  assert_success
+  assert_failure
 }
 
 @test "checking amavis: virusmail wiper cron exists" {
@@ -686,7 +656,7 @@ EOF
 @test "checking accounts: listmailuser (quotas enabled)" {
   run docker exec mail /bin/sh -c "sed -i '/ENABLE_QUOTAS=0/d' /etc/dms-settings; listmailuser | head -n 1"
   assert_success
-  assert_output '* user1@localhost.localdomain ( 12K / ~ ) [0%]'
+  assert_output '* user1@localhost.localdomain ( 10K / ~ ) [0%]'
 }
 
 @test "checking accounts: no error is generated when deleting a user if /tmp/docker-mailserver/postfix-accounts.cf is missing" {
@@ -987,11 +957,6 @@ EOF
   assert_success
 }
 
-@test "checking restart of process: clamd" {
-  run docker exec mail /bin/bash -c "pkill clamd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
-  assert_success
-}
-
 @test "checking restart of process: amavisd-new" {
   run docker exec mail /bin/bash -c "pkill amavi && sleep 12 && ps aux --forest | grep -v grep | grep '/usr/sbin/amavisd-new (master)'"
   assert_success