diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 0e4f1ab4..d862a2a6 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -14,6 +14,9 @@ jobs: run-tests: name: 'Test' runs-on: ubuntu-20.04 + strategy: + matrix: + part: [serial, parallel/set1, parallel/set2, parallel/set3] steps: - name: Checkout uses: actions/checkout@v3 @@ -45,6 +48,6 @@ jobs: cache-from: type=local,src=/tmp/.buildx-cache - name: 'Run tests' - run: make generate-accounts tests + run: make generate-accounts tests/${{ matrix.part }} env: CI: true diff --git a/Makefile b/Makefile index ce7ee47a..1577ab7c 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,12 @@ -SHELL := /bin/bash -.SHELLFLAGS += -e -u -o pipefail +SHELL := /bin/bash +.SHELLFLAGS += -e -u -o pipefail -export IMAGE_NAME := mailserver-testing:ci -export NAME ?= $(IMAGE_NAME) +PARALLEL_JOBS ?= 2 +export REPOSITORY_ROOT := $(CURDIR) +export IMAGE_NAME ?= mailserver-testing:ci +export NAME ?= $(IMAGE_NAME) + +.PHONY: ALWAYS_RUN # ----------------------------------------------- # --- Generic Targets --------------------------- @@ -17,7 +21,7 @@ build: --build-arg VCS_REVISION=$(shell cat VERSION) \ . -generate-accounts: +generate-accounts: ALWAYS_RUN @ cp test/config/templates/postfix-accounts.cf test/config/postfix-accounts.cf @ cp test/config/templates/dovecot-masters.cf test/config/dovecot-masters.cf @@ -33,14 +37,29 @@ clean: -@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && sudo rm -rf $${LINE}; done < .gitignore # ----------------------------------------------- -# --- Tests & Lints ---------------------------- +# --- Tests ------------------------------------ # ----------------------------------------------- -tests: - @ ./test/bats/bin/bats --timing test/*.bats +tests: ALWAYS_RUN +# See https://github.com/docker-mailserver/docker-mailserver/pull/2857#issuecomment-1312724303 +# on why `generate-accounts` is run before each set (TODO/FIXME) + @ $(MAKE) generate-accounts tests/serial + @ $(MAKE) generate-accounts tests/parallel/set1 + @ $(MAKE) generate-accounts tests/parallel/set2 + @ $(MAKE) generate-accounts tests/parallel/set3 -test/%: - @ ./test/bats/bin/bats --timing $@.bats +tests/serial: ALWAYS_RUN + @ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs 1 test/$@/**.bats + +tests/parallel/set%: ALWAYS_RUN + @ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs $(PARALLEL_JOBS) test/$@/**.bats + +test/%: ALWAYS_RUN + @ shopt -s globstar nullglob ; ./test/bats/bin/bats --timing test/tests/**/{$*,}.bats + +# ----------------------------------------------- +# --- Lints ------------------------------------- +# ----------------------------------------------- lint: eclint hadolint shellcheck diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md index da3b00fc..7270fcb8 100644 --- a/docs/content/contributing/general.md +++ b/docs/content/contributing/general.md @@ -24,6 +24,12 @@ To run the test suite, you will need to We do not support running linting, tests, etc on macOS at this time. Please use a linux VM. +??? tip "Running a Specific Test" + + To run a specific test, use `make build generate-accounts test/`, where `` is the file name of the test (_for more precision use a relative path: `test/test/`_) excluding the `.bats` suffix. + + To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`). + [Install Docker]: https://docs.docker.com/get-docker/ ## Documentation