diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 8c2d108c..e8168062 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -310,7 +310,7 @@ Enable or disable Rspamd. !!! warning "Current State" - Rspamd-support is under active development. Be aware that breaking changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. + Rspamd-support is under active development. Be aware that changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. - **0** => disabled - 1 => enabled @@ -335,6 +335,16 @@ The purpose of this setting is to opt-out of starting an internal Redis instance - 0 => Disabled - 1 => Enabled +##### RSPAMD_GREYLISTING + +Controls whether the [Rspamd Greylisting module][rspamd-greylisting-module] is enabled. This module can further assist in avoiding spam emails by [greylisting] e-mails with a certain spam score. + +- **0** => Disabled +- 1 => Enabled + +[rspamd-greylisting-module]: https://rspamd.com/doc/modules/greylisting.html +[greylisting]: https://en.wikipedia.org/wiki/Greylisting_(email) + ##### RSPAMD_LEARN When enabled, @@ -342,7 +352,7 @@ When enabled, 1. the "[autolearning][rspamd-autolearn]" feature is turned on; 2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). -!!! attention +!!! warning "Attention" As of now, the spam learning database is global (i.e. available to all users). If one user deliberately trains it with malicious data, then it will ruin your detection rate. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 20871c95..5f70fe73 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -4,7 +4,7 @@ title: 'Security | Rspamd' !!! warning "The current state of Rspamd integration into DMS" - Recent pull requests have stabilized integration of Rspamd to a point that we encourage users to test the feature. We are confident that there are no major bugs in our integration that make using Rspamd infeasible. Please note that there may still be (breaking) changes ahead as integration is still work in progress! + Recent pull requests have stabilized integration of Rspamd to a point that we encourage users to test the feature. We are confident that there are no major bugs in our integration that make using Rspamd infeasible. Please note that there may still be changes ahead as integration is still work in progress! We expect to stabilize this feature with version `v12.1.0`. @@ -16,16 +16,19 @@ If you want to have a look at the default configuration files for Rspamd that DM !!! note "AMD64 vs ARM64" - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference is two minor versions (AMD64 is at version 3.4, ARM64 at 3.2 \[13th Feb 2023\]). + We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 1st Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. - Maintainers noticed only few differences, some of them with a big impact though. For those running Rspamd on ARM64, we recommend [disabling](#with-the-help-of-a-custom-file) the [DKIM signing module][dkim-signing-module] if you don't use it. +## Related Environment Variables The following environment variables are related to Rspamd: 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) -3. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -4. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +3. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) +4. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +5. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) + +With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. ## The Default Configuration @@ -55,7 +58,7 @@ You can find a list of all Rspamd modules [on their website][modules]. #### Disabled By Default -DMS disables certain modules (clickhouse, elastic, greylist, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. +DMS disables certain modules (clickhouse, elastic, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. #### Anti-Virus (ClamAV) diff --git a/mailserver.env b/mailserver.env index 020d4050..de8a7653 100644 --- a/mailserver.env +++ b/mailserver.env @@ -133,6 +133,23 @@ ENABLE_RSPAMD=0 # 1 => Enabled ENABLE_RSPAMD_REDIS= +# When enabled, +# +# 1. the "[autolearning][rspamd-autolearn]" feature is turned on; +# 2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). +# +# **0** => disabled +# 1 => enabled +RSPAMD_LEARN=0 + +# Controls whether the Rspamd Greylisting module is enabled. +# This module can further assist in avoiding spam emails by greylisting +# e-mails with a certain spam score. +# +# **0** => disabled +# 1 => enabled +RSPAMD_GREYLISTING=0 + # Amavis content filter (used for ClamAV & SpamAssassin) # 0 => Disabled # 1 => Enabled diff --git a/target/rspamd/local.d/greylist.conf b/target/rspamd/local.d/greylist.conf new file mode 100644 index 00000000..4098ee8f --- /dev/null +++ b/target/rspamd/local.d/greylist.conf @@ -0,0 +1,3 @@ +# documentation: https://www.rspamd.com/doc/modules/greylisting.html + +enabled = false; diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index ff550dee..039c8f74 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -130,9 +130,10 @@ function _register_functions [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' - [[ ${ENABLE_RSPAMD} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd' + + # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' + [[ ${ENABLE_RSPAMD} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd' # needs to be started before SASLauthd [[ ${ENABLE_OPENDKIM} -eq 1 ]] && _register_start_daemon '_start_daemon_opendkim' diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 8b478904..ba292346 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -1,24 +1,23 @@ #!/bin/bash +# Function called during global setup to handle the complete setup of Rspamd. function _setup_rspamd { if [[ ${ENABLE_RSPAMD} -eq 1 ]] then - _log 'warn' 'Rspamd integration is work in progress - expect (breaking) changes at any time' + _log 'warn' 'Rspamd integration is work in progress - expect changes at any time' _log 'debug' 'Enabling and configuring Rspamd' - __rspamd__preflight_checks_and_setup - __rspamd__adjust_postfix_configuration - __rspamd__disable_default_modules - __rspamd__handle_modules_configuration + __rspamd__run_early_setup_and_checks # must run first + __rspamd__setup_redis + __rspamd__setup_postfix + __rspamd__setup_clamav + __rspamd__setup_default_modules + __rspamd__setup_learning + __rspamd__setup_greylisting + __rspamd__handle_user_modules_adjustments # must run last - if [[ ${RSPAMD_LEARN} -eq 1 ]] - then - __rspamd__log 'debug' 'Enabling and configuring learning' - __rspamd__setup_learning - else - __rspamd__log 'debug' 'Learning is disabled' - fi + _log 'trace' 'Rspamd setup finished' else _log 'debug' 'Rspamd is disabled' fi @@ -31,75 +30,13 @@ function _setup_rspamd # @param ${2} = message function __rspamd__log { _log "${1:-}" "(Rspamd setup) ${2:-}" ; } -# Run miscellaneous checks against the current configuration so we can -# properly handle integration into ClamAV, etc. -# -# This will also check whether Amavis is enabled and emit a warning as -# we discourage users from running Amavis & Rspamd at the same time. -function __rspamd__preflight_checks_and_setup -{ - touch /var/lib/rspamd/stats.ucl - - if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] - then - __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' - fi - - if [[ ${ENABLE_CLAMAV} -eq 1 ]] - then - __rspamd__log 'debug' 'Enabling ClamAV integration' - sedfile -i -E 's|^(enabled).*|\1 = true;|g' /etc/rspamd/local.d/antivirus.conf - # RSpamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group - usermod -a -G clamav _rspamd - else - __rspamd__log 'debug' 'Rspamd will not use ClamAV (which has not been enabled)' - fi - - if [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] - then - __rspamd__log 'trace' 'Internal Redis is enabled, adding configuration' - cat >/etc/rspamd/local.d/redis.conf << "EOF" -# documentation: https://rspamd.com/doc/configuration/redis.html - -servers = "127.0.0.1:6379"; -expand_keys = true; - -EOF - - # Here we adjust the Redis default configuration that we supply to Redis - # when starting it. Note that `/var/lib/redis/` is linked to - # `/var/mail-state/redis/` (for persisting it) if `ONE_DIR=1`. - sedfile -i -E \ - -e 's|^(bind).*|\1 127.0.0.1|g' \ - -e 's|^(daemonize).*|\1 no|g' \ - -e 's|^(port).*|\1 6379|g' \ - -e 's|^(loglevel).*|\1 warning|g' \ - -e 's|^(logfile).*|\1 ""|g' \ - -e 's|^(dir).*|\1 /var/lib/redis|g' \ - -e 's|^(dbfilename).*|\1 dms-dump.rdb|g' \ - /etc/redis/redis.conf - else - __rspamd__log 'debug' 'Rspamd will not use internal Redis (which has been disabled)' - fi -} - -# Adjust Postfix's configuration files. Append Rspamd at the end of -# `smtpd_milters` in `main.cf`. -function __rspamd__adjust_postfix_configuration -{ - postconf 'rspamd_milter = inet:localhost:11332' - - # shellcheck disable=SC2016 - sed -i -E 's|^(smtpd_milters =.*)|\1 \$rspamd_milter|g' /etc/postfix/main.cf -} - # Helper for explicitly enabling or disabling a specific module. # # @param ${1} = module name # @param ${2} = `true` when you want to enable the module (default), # `false` when you want to disable the module [OPTIONAL] # @param ${3} = whether to use `local` (default) or `override` [OPTIONAL] -function __rspamd__enable_disable_module +function __rspamd__helper__enable_disable_module { local MODULE=${1:?Module name must be provided} local ENABLE_MODULE=${2:-true} @@ -108,7 +45,7 @@ function __rspamd__enable_disable_module if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]] then - __rspamd__log 'warn' "__rspamd__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" + __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" return 1 fi @@ -123,16 +60,87 @@ enabled = ${ENABLE_MODULE}; EOF } +# Run miscellaneous early setup tasks and checks, such as creating files needed at runtime +# or checking for other anti-spam/anti-virus software. +function __rspamd__run_early_setup_and_checks +{ + mkdir -p /var/lib/rspamd/ + : >/var/lib/rspamd/stats.ucl + + if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] + then + __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' + fi +} + +# Sets up Redis. In case the user does not use a dedicated Redis instance, we +# supply a configuration for our local Redis instance which is started later. +function __rspamd__setup_redis +{ + if [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] + then + __rspamd__log 'debug' 'Internal Redis is enabled, adding configuration' + cat >/etc/rspamd/local.d/redis.conf << "EOF" +# documentation: https://rspamd.com/doc/configuration/redis.html + +servers = "127.0.0.1:6379"; +expand_keys = true; + +EOF + + # Here we adjust the Redis default configuration that we supply to Redis + # when starting it. Note that `/var/lib/redis/` is linked to + # `/var/mail-state/redis/` (for persisting it) if `ONE_DIR=1`. + sedfile -i -E \ + -e 's|^(bind).*|\1 127.0.0.1|g' \ + -e 's|^(daemonize).*|\1 no|g' \ + -e 's|^(port).*|\1 6379|g' \ + -e 's|^(loglevel).*|\1 warning|g' \ + -e 's|^(logfile).*|\1 ""|g' \ + -e 's|^(dir).*|\1 /var/lib/redis|g' \ + -e 's|^(dbfilename).*|\1 dms-dump.rdb|g' \ + /etc/redis/redis.conf + else + __rspamd__log 'debug' 'Rspamd will not use internal Redis (which has been disabled)' + fi +} + +# Adjust Postfix's configuration files. We only need to append Rspamd at the end of +# `smtpd_milters` in `/etc/postfix/main.cf`. +function __rspamd__setup_postfix +{ + __rspamd__log 'debug' "Adjusting Postfix's configuration" + + postconf 'rspamd_milter = inet:localhost:11332' + # shellcheck disable=SC2016 + sed -i -E 's|^(smtpd_milters =.*)|\1 \$rspamd_milter|g' /etc/postfix/main.cf +} + +# If ClamAV is enabled, we will integrate it into Rspamd. +function __rspamd__setup_clamav +{ + if [[ ${ENABLE_CLAMAV} -eq 1 ]] + then + __rspamd__log 'debug' 'Enabling ClamAV integration' + sedfile -i -E 's|^(enabled).*|\1 = true;|g' /etc/rspamd/local.d/antivirus.conf + # Rspamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group + usermod -a -G clamav _rspamd + else + __rspamd__log 'debug' 'Rspamd will not use ClamAV (which has not been enabled)' + fi +} + # Disables certain modules by default. This can be overwritten by the user later. # We disable the modules listed in `DISABLE_MODULES` as we believe these modules # are not commonly used and the average user does not need them. As a consequence, # disabling them saves resources. -function __rspamd__disable_default_modules +function __rspamd__setup_default_modules { + __rspamd__log 'debug' 'Disabling default modules' + local DISABLE_MODULES=( clickhouse elastic - greylist neural reputation spamassassin @@ -142,14 +150,74 @@ function __rspamd__disable_default_modules for MODULE in "${DISABLE_MODULES[@]}" do - __rspamd__enable_disable_module "${MODULE}" 'false' + __rspamd__helper__enable_disable_module "${MODULE}" 'false' done } +# This function sets up intelligent learning of Junk, by +# +# 1. enabling auto-learn for the classifier-bayes module +# 2. setting up sieve scripts that detect when a user is moving e-mail +# from or to the "Junk" folder, and learning them as ham or spam. +function __rspamd__setup_learning +{ + if [[ ${RSPAMD_LEARN} -eq 1 ]] + then + __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' + + local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" + + sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf + sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf + cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF + + # From elsewhere to Junk folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY + imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve + + # From Junk folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve +} +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; +EOF + + sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" + sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" + else + __rspamd__log 'debug' 'Intelligent learning of spam and ham is disabled' + fi +} + +# Sets up greylisting based on the environment variable RSPAMD_GREYLISTING. +function __rspamd__setup_greylisting +{ + if [[ ${RSPAMD_GREYLISTING} -eq 1 ]] + then + __rspamd__log 'debug' 'Enabling greylisting' + sedfile -i -E "s|(enabled =).*|\1 true;|g" /etc/rspamd/local.d/greylist.conf + else + __rspamd__log 'debug' 'Greylisting is disabled' + fi +} + # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file -function __rspamd__handle_modules_configuration +function __rspamd__handle_user_modules_adjustments { # Adds an option with a corresponding value to a module, or, in case the option # is already present, overwrites it. @@ -161,7 +229,7 @@ function __rspamd__handle_modules_configuration # # ## Note # - # While this function is currently bound to the scope of `__rspamd__handle_modules_configuration`, + # While this function is currently bound to the scope of `__rspamd__handle_user_modules_adjustments`, # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` # are set) so that it may be used elsewhere if needed. function __add_or_replace @@ -196,11 +264,11 @@ function __rspamd__handle_modules_configuration case "${COMMAND}" in ('disable-module') - __rspamd__enable_disable_module "${ARGUMENT1}" 'false' 'override' + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' ;; ('enable-module') - __rspamd__enable_disable_module "${ARGUMENT1}" 'true' 'override' + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' ;; ('set-option-for-module') @@ -233,46 +301,3 @@ function __rspamd__handle_modules_configuration done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi } - -# This function sets up intelligent learning of Junk, by -# -# 1. enabling auto-learn for the classifier-bayes module -# 2. setting up sieve scripts that detect when a user is moving e-mail -# from or to the "Junk" folder, and learning them as ham or spam. -function __rspamd__setup_learning -{ - __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' - - local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' - ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" - - sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf - sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf - cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF - - # From elsewhere to Junk folder - imapsieve_mailbox1_name = Junk - imapsieve_mailbox1_causes = COPY - imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve - - # From Junk folder to elsewhere - imapsieve_mailbox2_name = * - imapsieve_mailbox2_from = Junk - imapsieve_mailbox2_causes = COPY - imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve -} -EOF - - cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF -require ["vnd.dovecot.pipe", "copy", "imapsieve"]; -pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; -EOF - - cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF -require ["vnd.dovecot.pipe", "copy", "imapsieve"]; -pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; -EOF - - sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" - sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" -} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 4611dba4..5bf5a340 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -55,6 +55,7 @@ function __environment_variables_general_setup VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index 658a0e68..b0d0f88b 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -18,6 +18,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 --env RSPAMD_LEARN=1 + --env RSPAMD_GREYLISTING=1 ) mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" @@ -243,3 +244,11 @@ function teardown_file() { _default_teardown ; } assert_output --partial "${LINE}" done } + +@test 'Check greylisting is enabled' { + _run_in_container grep 'enabled = true;' /etc/rspamd/local.d/greylist.conf + assert_success + _run_in_container rspamadm configdump greylist + assert_success + assert_output --partial 'enabled = true;' +}