From 2cd534a1ab1f2998432e9f196f6d053385f7975a Mon Sep 17 00:00:00 2001
From: Brennan Kinney <5098581+polarathene@users.noreply.github.com>
Date: Sat, 26 Nov 2022 10:58:16 +1300
Subject: [PATCH] tests(CI): Adjust Makefile & GHA workflow to support new test
 layout

These updates support running tests that have been relocated into `serial` and `parallel/set*` directories.

- `make tests` now calls the two make targets beneath it. The only difference is that `serial` continues the "1 test at a time" approach used prior to this PR, while the `parallel` target increases the `--jobs` arg to run multiple tests concurrently (_configured by `PARALLEL_JOBS`_).
- The `test/%` target leverages Bash syntax magic to ease running single tests without providing the exact path.
- This syntax also supports providing multiple test names (eg: `make test/clamav,template`) to run.
- `**` (globstar) allows for future improvements that can group multiple test files into sub-directories by their scope (eg: anti-spam, ssl, etc).

---

chore: Add `shopt -s globstar` to other targets
I realized that other targets should have this as well in case it is not set.
It is better to be more explicit here than to have weird errors due to `**` not expanding properly.

---

fix(Makefile):  Add back `.PHONY` targets

I encountered `make` telling me the target was already up-to-date, which of course is nonsense.
I therefore added back the `.PHONY` targets to ensure tests are always run.

---

docs: Added instructions for running  a single test
See https://github.com/docker-mailserver/docker-mailserver/pull/2857/files#r1008582760
---
 .github/workflows/generic_test.yml   |  5 +++-
 Makefile                             | 39 +++++++++++++++++++++-------
 docs/content/contributing/general.md |  6 +++++
 3 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml
index 0e4f1ab4..d862a2a6 100644
--- a/.github/workflows/generic_test.yml
+++ b/.github/workflows/generic_test.yml
@@ -14,6 +14,9 @@ jobs:
   run-tests:
     name: 'Test'
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        part: [serial, parallel/set1, parallel/set2, parallel/set3]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -45,6 +48,6 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
 
       - name: 'Run tests'
-        run: make generate-accounts tests
+        run: make generate-accounts tests/${{ matrix.part }}
         env:
           CI: true
diff --git a/Makefile b/Makefile
index ce7ee47a..1577ab7c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,12 @@
-SHELL             := /bin/bash
-.SHELLFLAGS       += -e -u -o pipefail
+SHELL       := /bin/bash
+.SHELLFLAGS += -e -u -o pipefail
 
-export IMAGE_NAME := mailserver-testing:ci
-export NAME       ?= $(IMAGE_NAME)
+PARALLEL_JOBS          ?= 2
+export REPOSITORY_ROOT := $(CURDIR)
+export IMAGE_NAME      ?= mailserver-testing:ci
+export NAME            ?= $(IMAGE_NAME)
+
+.PHONY: ALWAYS_RUN
 
 # -----------------------------------------------
 # --- Generic Targets ---------------------------
@@ -17,7 +21,7 @@ build:
 		--build-arg VCS_REVISION=$(shell cat VERSION) \
 		.
 
-generate-accounts:
+generate-accounts: ALWAYS_RUN
 	@ cp test/config/templates/postfix-accounts.cf test/config/postfix-accounts.cf
 	@ cp test/config/templates/dovecot-masters.cf test/config/dovecot-masters.cf
 
@@ -33,14 +37,29 @@ clean:
 	-@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && sudo rm -rf $${LINE}; done < .gitignore
 
 # -----------------------------------------------
-# --- Tests  & Lints ----------------------------
+# --- Tests  ------------------------------------
 # -----------------------------------------------
 
-tests:
-	@ ./test/bats/bin/bats --timing test/*.bats
+tests: ALWAYS_RUN
+# See https://github.com/docker-mailserver/docker-mailserver/pull/2857#issuecomment-1312724303
+# on why `generate-accounts` is run before each set (TODO/FIXME)
+	@ $(MAKE) generate-accounts tests/serial
+	@ $(MAKE) generate-accounts tests/parallel/set1
+	@ $(MAKE) generate-accounts tests/parallel/set2
+	@ $(MAKE) generate-accounts tests/parallel/set3
 
-test/%:
-	@ ./test/bats/bin/bats --timing $@.bats
+tests/serial: ALWAYS_RUN
+	@ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs 1 test/$@/**.bats
+
+tests/parallel/set%: ALWAYS_RUN
+	@ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs $(PARALLEL_JOBS) test/$@/**.bats
+
+test/%: ALWAYS_RUN
+	@ shopt -s globstar nullglob ; ./test/bats/bin/bats --timing test/tests/**/{$*,}.bats
+
+# -----------------------------------------------
+# --- Lints -------------------------------------
+# -----------------------------------------------
 
 lint: eclint hadolint shellcheck
 
diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md
index da3b00fc..7270fcb8 100644
--- a/docs/content/contributing/general.md
+++ b/docs/content/contributing/general.md
@@ -24,6 +24,12 @@ To run the test suite, you will need to
 
     We do not support running linting, tests, etc on macOS at this time. Please use a linux VM.
 
+??? tip "Running a Specific Test"
+
+    To run a specific test, use `make build generate-accounts test/<TEST NAME>`, where `<TEST NAME>` is the file name of the test (_for more precision use a relative path: `test/test/<PATH>`_) excluding the `.bats` suffix.
+
+    To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`).
+
 [Install Docker]: https://docs.docker.com/get-docker/
 
 ## Documentation