From cd1721334ce4e7a6cd21f7e7f3a84f2a32244db7 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:02:56 +0200 Subject: [PATCH] scripts: Rspamd stabilization pt. 2 (#3282) * move modules adjustment file to new location Because we link `/tmp/docker-mailserver/rspamd/override.d` to `/etc/rspamd/override.d`, I think it makes sense to move the modules adjustment file into `/tmp/docker-mailserver/rspamd/` as well. I write the code in a way that it is backwards compatible for now, so this is NOT a breaking change. * minor improvement to `__rspamd__handle_user_modules_adjustments` The expansion of `ARGUMENT3` is now done in a way that only adds the whitespace in case the variable is set and not null. * move test file structure to respect latest changes Because we're now linking `rspamd/override.d/`, we can simplify the setup a bit. But this requires a change in directory structure. The current Rspamd test will be renamed to `rspamd_full.bats`, because I plan on adding more tests in different files for different feature sets. This is done to make this feature well-tested! * improved and added tests to Rspamd-full FYI: The line ```bats _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" ``` was testing a condition that should actually not be met, but when I started working on this feature, I thought this was the correct configuration. Adding the `assert_success` statements revealed this wrong line. I also added tests to check whether `override.d` is linked correctly. * renamed: `rspamd.bats` => `rspamd_full.bats` * added new tests for incomplete Rspamd feature set We now test that warnings are emitted & features are disabled correctly. * update documentation --- .../config/advanced/optional-config.md | 2 +- docs/content/config/security/rspamd.md | 14 ++-- .../startup/setup.d/security/rspamd.sh | 19 ++++- test/config/rspamd/user-patches.sh | 13 --- .../postfix-accounts.cf | 0 .../rspamd/custom-commands.conf} | 0 .../override.d/testmodule_complicated.conf | 3 + test/config/rspamd_full/user-patches.sh | 16 ++++ .../{rspamd.bats => rspamd_full.bats} | 35 ++++++-- .../set1/spam_virus/rspamd_partly.bats | 84 +++++++++++++++++++ 10 files changed, 155 insertions(+), 31 deletions(-) delete mode 100644 test/config/rspamd/user-patches.sh rename test/config/{rspamd => rspamd_full}/postfix-accounts.cf (100%) rename test/config/{rspamd-modules.conf => rspamd_full/rspamd/custom-commands.conf} (100%) create mode 100644 test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf create mode 100644 test/config/rspamd_full/user-patches.sh rename test/tests/parallel/set1/spam_virus/{rspamd.bats => rspamd_full.bats} (89%) create mode 100644 test/tests/parallel/set1/spam_virus/rspamd_partly.bats diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 45a717f5..a5266681 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -40,7 +40,7 @@ This is a list of all configuration files and directories which are optional or - **dovecot.cf:** replaces `/etc/dovecot/local.conf`. (Docs: [Override Dovecot Defaults][docs-override-dovecot]) - **dovecot-quotas.cf:** list of custom quotas per mailbox. (Docs: [Accounts][docs-accounts-quota]) - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) -- **rspamd-commands:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) +- **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index b4b48f79..f017ca7f 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -16,7 +16,7 @@ If you want to have a look at the default configuration files for Rspamd that DM !!! note "AMD64 vs ARM64" - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 1st Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. + We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 23rd Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. [rspamd-homepage]: https://rspamd.com/ [dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd @@ -117,7 +117,7 @@ If you want to overwrite the default settings and / or provide your own settings ### With the Help of a Custom File -DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `rspamd-modules.conf` into the [local config directory `docker-data/dms/config/`][docs-volumes-config]. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: +DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `custom-commands.conf` into `docker-data/dms/config/rspamd/`. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: ```txt COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 @@ -139,13 +139,12 @@ where `COMMAND` can be: For command 1 - 3, we append the `.conf` suffix to the module name to get the correct file name automatically. For commands 4 - 6, the file name is fixed (you don't even need to provide it). For command 7, you will need to provide the whole file name (including the suffix) yourself! -You can also have comments (the line starts with `#`) and blank lines in `rspamd-modules.conf` - they are properly handled and not evaluated. +You can also have comments (the line starts with `#`) and blank lines in `custom-commands.conf` - they are properly handled and not evaluated. !!! tip "Adjusting Modules This Way" These simple commands are meant to give users the ability to _easily_ alter modules and their options. As a consequence, they are not powerful enough to enable multi-line adjustments. If you need to do something more complex, we advise to do that [manually](#manually)! -[docs-volumes-config]: ../advanced/optional-config.md [rspamd-docs-basic-options]: https://rspamd.com/doc/configuration/options.html ## Examples & Advanced Configuration @@ -169,10 +168,9 @@ This will enable Rspamd and disable services you don't need when using Rspamd. N Rspamd is running, but you want or need to adjust it? -1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: create a file called `rspamd-modules.conf` and add the line `set-option-for-controller secure_ip "0.0.0.0/0"`. Place the file `rspamd-modules.conf` inside the directory on the host you mount to `/tmp/docker-mailserver/` inside the container (in our documentation, we use `docker-data/dms/config` on the host for this purpose). And you're done! Note: this disables authentication on the website - make sure you know what you're doing! -2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem, just add another line to `rspamd-modules.conf` that looks like this: `set-option-for-module classifier-bayes autolearn true`. -3. But the chartable module gets on your nerves? Just disable it by adding another line: `disable-module chartable -`. +1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: create a file called `custom-commands.conf` and add the line `set-option-for-controller secure_ip "0.0.0.0/0"`. Place the file `custom-commands.conf` inside the directory on the host you mount to `/tmp/docker-mailserver/rspamd/` inside the container (in our documentation, we use `docker-data/dms/config/rspamd/` on the host for this purpose). And you're done! Note: this disables authentication on the website - make sure you know what you're doing! +2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem, just add another line to `custom-commands.conf` that looks like this: `set-option-for-module classifier-bayes autolearn true`. +3. But the chartable module gets on your nerves? Just disable it by adding another line: `disable-module chartable`. ### DKIM Signing diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index bd7f03e0..cee7c2a9 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -292,7 +292,7 @@ function __rspamd__handle_user_modules_adjustments # Adds an option with a corresponding value to a module, or, in case the option # is already present, overwrites it. # - # @param ${1} = file name in /etc/rspamd/override.d/ + # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ # @param ${2} = module name as it should appear in the log # @patam ${3} = option name in the module # @param ${4} = value of the option @@ -324,10 +324,21 @@ function __rspamd__handle_user_modules_adjustments fi } - local RSPAMD_CUSTOM_COMMANDS_FILE='/tmp/docker-mailserver/rspamd-modules.conf' + local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" + local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" + + # We check for usage of the previous location of the commands file. + # This can be removed after the release of v14.0.0. + if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]] + then + __rspamd__log 'warn' "Detected usage of old file location for modules adjustment ('${RSPAMD_CUSTOM_COMMANDS_FILE_OLD}') - please use the new location ('${RSPAMD_CUSTOM_COMMANDS_FILE}')" + __rspamd__log 'warn' "Using old file location now (deprecated) - this will prevent startup in v13.0.0" + RSPAMD_CUSTOM_COMMANDS_FILE=${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} + fi + if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]] then - __rspamd__log 'debug' "Found file 'rspamd-modules.conf' - parsing and applying it" + __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 do @@ -359,7 +370,7 @@ function __rspamd__handle_user_modules_adjustments ('add-line') __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" - echo "${ARGUMENT2} ${ARGUMENT3:-}" >>"/etc/rspamd/override.d/${ARGUMENT1}" + echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" ;; (*) diff --git a/test/config/rspamd/user-patches.sh b/test/config/rspamd/user-patches.sh deleted file mode 100644 index 00e6ca97..00000000 --- a/test/config/rspamd/user-patches.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -cat >/etc/rspamd/override.d/testmodule_complicated.conf << EOF -complicated { - anOption = someValue; -} -EOF - -echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc - -echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf -sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf -echo -e 'sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/config/rspamd/postfix-accounts.cf b/test/config/rspamd_full/postfix-accounts.cf similarity index 100% rename from test/config/rspamd/postfix-accounts.cf rename to test/config/rspamd_full/postfix-accounts.cf diff --git a/test/config/rspamd-modules.conf b/test/config/rspamd_full/rspamd/custom-commands.conf similarity index 100% rename from test/config/rspamd-modules.conf rename to test/config/rspamd_full/rspamd/custom-commands.conf diff --git a/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf b/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf new file mode 100644 index 00000000..212b4fb3 --- /dev/null +++ b/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf @@ -0,0 +1,3 @@ +complicated { + anOption = someValue; +} diff --git a/test/config/rspamd_full/user-patches.sh b/test/config/rspamd_full/user-patches.sh new file mode 100644 index 00000000..9f4dc9fb --- /dev/null +++ b/test/config/rspamd_full/user-patches.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Enable GTUBE test patterns so we can properly check whether +# Rspamd is rejecting mail, adding headers, etc. +# +# We do not use `custom-commands.conf` because this a feature +# we are testing too. +echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc + +# We want Dovecot to be very detailed about what it is doing, +# specificially for Sieve because we need to check whether the +# Sieve scripts are executed so Rspamd is trained when using +# `RSPAMD_LEARN=1`. +echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf +sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf +echo -e '\n sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats similarity index 89% rename from test/tests/parallel/set1/spam_virus/rspamd.bats rename to test/tests/parallel/set1/spam_virus/rspamd_full.bats index 2a4f15f0..d0c22a8b 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -1,8 +1,10 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" -BATS_TEST_NAME_PREFIX='[Rspamd] ' -CONTAINER_NAME='dms-test_rspamd' +# This file tests Rspamd when all of its features are enabled, and +# all other interfering features are disabled. +BATS_TEST_NAME_PREFIX='[Rspamd] (full) ' +CONTAINER_NAME='dms-test_rspamd-full' function setup_file() { _init_with_defaults @@ -27,7 +29,7 @@ function setup_file() { --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 ) - mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" + cp -r "${TEST_TMP_CONFIG}"/rspamd_full/* "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' # wait for ClamAV to be fully setup or we will get errors on the log @@ -62,7 +64,28 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } -@test 'logs exist and contains proper content' { +@test "'/etc/rspamd/override.d/' is linked correctly" { + local OVERRIDE_D='/etc/rspamd/override.d' + + _run_in_container_bash "[[ -h ${OVERRIDE_D} ]]" + assert_success + + _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" + assert_success +} + +@test 'startup log shows all features as properly enabled' { + run docker logs "${CONTAINER_NAME}" + assert_success + assert_line --partial 'Enabling ClamAV integration' + assert_line --partial 'Setting up intelligent learning of spam and ham' + assert_line --partial 'Enabling greylisting' + assert_line --partial 'Hfilter (group) module is enabled' + assert_line --partial "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to" + assert_line --partial "Found file '/tmp/docker-mailserver/rspamd/custom-commands.conf' - parsing and applying it" +} + +@test 'service log exist and contains proper content' { _service_log_should_contain_string 'rspamd' 'rspamd .* is loading configuration' _service_log_should_contain_string 'rspamd' 'lua module clickhouse is disabled in the configuration' _service_log_should_contain_string 'rspamd' 'lua module elastic is disabled in the configuration' @@ -200,10 +223,12 @@ function teardown_file() { _default_teardown ; } done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + assert_success local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' _run_in_container grep 'sieve_plugins.*sieve_imapsieve' "${SIEVE_CONFIG_FILE}" - _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" + assert_success _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}" + assert_success # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats new file mode 100644 index 00000000..29b7bc0e --- /dev/null +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -0,0 +1,84 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +# This file tests Rspamd when some of its features are enabled, and +# some other interfering features are enabled. +BATS_TEST_NAME_PREFIX='[Rspamd] (partly) ' +CONTAINER_NAME='dms-test_rspamd-partly' + +function setup_file() { + _init_with_defaults + + # Comment for maintainers about `PERMIT_DOCKER=host`: + # https://github.com/docker-mailserver/docker-mailserver/pull/2815/files#r991087509 + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env ENABLE_CLAMAV=0 + --env ENABLE_RSPAMD=1 + --env ENABLE_OPENDKIM=1 + --env ENABLE_OPENDMARC=1 + --env ENABLE_POLICYD_SPF=1 + --env ENABLE_POSTGREY=0 + --env PERMIT_DOCKER=host + --env LOG_LEVEL=trace + --env MOVE_SPAM_TO_JUNK=0 + --env RSPAMD_LEARN=0 + --env RSPAMD_GREYLISTING=0 + --env RSPAMD_HFILTER=0 + ) + + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + _wait_for_service rspamd-redis + _wait_for_service rspamd + _wait_for_service amavis + _wait_for_service postfix + _wait_for_smtp_port_in_container +} + +function teardown_file() { _default_teardown ; } + +@test "log warns about interfering features" { + run docker logs "${CONTAINER_NAME}" + assert_success + for SERVICE in 'Amavis/SA' 'OpenDKIM' 'OpenDMARC' 'policyd-spf' + do + assert_output --regexp ".*WARNING.*Running ${SERVICE} & Rspamd at the same time is discouraged" + done +} + +@test 'log shows all features as properly disabled' { + run docker logs "${CONTAINER_NAME}" + assert_success + assert_line --partial 'Rspamd will not use ClamAV (which has not been enabled)' + assert_line --partial 'Intelligent learning of spam and ham is disabled' + assert_line --partial 'Greylisting is disabled' + assert_line --partial 'Disabling Hfilter (group) module' +} + +@test 'learning is properly disabled' { + for FILE in learn-{ham,spam}.{sieve,svbin} + do + _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" + assert_failure + done + + _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + assert_failure + local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' + _run_in_container grep -F 'imapsieve_mailbox1_name = Junk' "${SIEVE_CONFIG_FILE}" + assert_failure + _run_in_container grep -F 'imapsieve_mailbox1_causes = COPY' "${SIEVE_CONFIG_FILE}" + assert_failure +} + +@test 'greylisting is properly disabled' { + _run_in_container grep -F 'enabled = false;' '/etc/rspamd/local.d/greylist.conf' + assert_success +} + +@test 'hfilter group module configuration is deleted' { + _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + assert_failure +}