From 265440b2bbc77971b579bd00ad92a89e8c02d9f3 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:34:15 +1300 Subject: [PATCH 1/5] fix: Ensure `.svbin` files are newer than `.sieve` source files (#3779) --- CHANGELOG.md | 3 ++- target/scripts/startup/setup-stack.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ade3c9b..31c4819f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ All notable changes to this project will be documented in this file. The format - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) - ### Fixes +- **Dovecot:** + - During container startup for Dovecot Sieve, `.sievec` source files compiled to `.svbin` now have their `mtime` adjusted post setup to ensure it is always older than the associated `.svbin` file. This avoids superfluous error logs for sieve scripts that don't actually need to be compiled again ([#3779](https://github.com/docker-mailserver/docker-mailserver/pull/3779)) - **Internal:** - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) - Fix missing 'jaq' binary for ARM architecture ([#3766](https://github.com/docker-mailserver/docker-mailserver/pull/3766)) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 060dadb2..f55cb548 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -20,6 +20,16 @@ function _setup() { ${FUNC} done + _setup_post +} + +function _setup_post() { + # Dovecot `.svbin` files must have a newer mtime than their `.sieve` source files, + # Modifications during setup to these files sometimes results in a common mtime value. + # Handled during post-setup as setup of Dovecot Sieve scripts is not centralized. + find /usr/lib/dovecot/ -iname '*.sieve' -exec touch -d '2 seconds ago' {} + + find /usr/lib/dovecot/ -iname '*.svbin' -exec touch -d '1 seconds ago' {} + + # All startup modifications to configs should have taken place before calling this: _prepare_for_change_detection } From 2bf52342500726e86362ee1ea5954b7d96533d32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:18:13 +1300 Subject: [PATCH 2/5] chore(deps): Bump anchore/scan-action from 3.4.0 to 3.5.0 (#3782) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d7cf6932..896ee80e 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.4.0 + uses: anchore/scan-action@v3.5.0 id: scan with: image: mailserver-testing:ci From 068ceb1d1aaba8b682bf0be34dbc2a134d823643 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:38:08 +0100 Subject: [PATCH 3/5] docs: misc improvements (#3773) * correct misc typos We also seem to be favoring `behavior` over `behaviour`. * bump MkDocs version * resolve errors shown when buildg docs * improve the Rspamd page * behaviour -> behavior Streamline the usage of this word. The majority used behavior, so I opted to go with this way of spelling it. * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/scripts/docs/build-docs.sh | 2 +- CHANGELOG.md | 1 + docs/content/config/advanced/auth-ldap.md | 2 +- docs/content/config/advanced/ipv6.md | 6 +- .../config/advanced/optional-config.md | 2 +- docs/content/config/advanced/podman.md | 2 +- docs/content/config/debugging.md | 4 +- docs/content/config/environment.md | 4 +- docs/content/config/security/rspamd.md | 61 ++++++++++++++++--- .../examples/use-cases/imap-folders.md | 2 +- docs/content/faq.md | 6 +- docs/content/index.md | 10 +-- docs/mkdocs.yml | 4 +- 13 files changed, 77 insertions(+), 29 deletions(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index aff66ab5..5d1cab52 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.2 build --strict + squidfunk/mkdocs-material:9.5 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c4819f..b871f0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. The format - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) + - Detailed how mail received is assigned a spam score by Rspamd and processed accordingly ([#3773](https://github.com/docker-mailserver/docker-mailserver/pull/3773)) ### Fixes diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index c275fd6c..ea24526d 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -56,7 +56,7 @@ These variables specify the LDAP filters that dovecot uses to determine if a use This is split into the following two lookups, both using `%u` as the placeholder for the full login name ([see dovecot documentation for a full list of placeholders](https://doc.dovecot.org/configuration_manual/config_file/config_variables/)). Usually you only need to set `DOVECOT_USER_FILTER`, in which case it will be used for both filters. - `DOVECOT_USER_FILTER` is used to get the account details (uid, gid, home directory, quota, ...) of a user. -- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behaviour if left away). +- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behavior if left away). If your directory doesn't have the [postfix-book schema](https://github.com/variablenix/ldap-mail-schema/blob/master/postfix-book.schema) installed, then you must change the internal attribute handling for dovecot. For this you have to change the `pass_attr` and the `user_attr` mapping, as shown in the example below: diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index f035523a..77dfb0c9 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -60,7 +60,7 @@ Enable `ip6tables` support so that Docker will manage IPv6 networking rules as w ``` - `experimental: true` is currently required for `ip6tables: true` to work. - - `userland-proxy` setting [can potentially affect connection behaviour][gh-pull-3244-proxy] for local connections. + - `userland-proxy` setting [can potentially affect connection behavior][gh-pull-3244-proxy] for local connections. Now restart the daemon if it's running: `systemctl restart docker`. @@ -99,7 +99,7 @@ Next, configure a network with an IPv6 subnet for your container with any of the ??? tip "Override the implicit `default` network" - You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates](docker-docs-network-compose-default). Just replace `dms-ipv6` with `default`. + You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates][docker-docs-network-compose-default]. Just replace `dms-ipv6` with `default`. The Docker Compose `default` bridge is not affected by settings for the default `bridge` (aka `docker0`) in `/etc/docker/daemon.json`. @@ -180,7 +180,7 @@ curl --max-time 5 http://[2001:db8::1]:80 !!! info "IPv6 ULA address priority" DNS lookups that have records for both IPv4 and IPv6 addresses (_eg: `localhost`_) may prefer IPv4 over IPv6 (ULA) for private addresses, whereas for public addresses IPv6 has priority. This shouldn't be anything to worry about, but can come across as a surprise when testing your IPv6 setup on the same host instead of from a remote client. - + The preference can be controlled with [`/etc/gai.conf`][networking-gai], and appears was configured this way based on [the assumption that IPv6 ULA would never be used with NAT][networking-gai-blog]. It should only affect the destination resolved for outgoing connections, which for IPv6 ULA should only really affect connections between your containers / host. In future [IPv6 ULA may also be prioritized][networking-gai-rfc]. [docker-subnets]: https://straz.to/2021-09-08-docker-address-pools/#what-are-the-default-address-pools-when-no-configuration-is-given-vanilla-pools diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 21f82a3b..31090050 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -35,7 +35,7 @@ This is a list of all configuration files and directories which are optional or - **whitelist_clients.local:** Whitelisted domains, not considered by postgrey. Enter one host or domain per line. - **spamassassin-rules.cf:** Anti-spam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) - **fail2ban-fail2ban.cf:** Additional config options for `fail2ban.cf`. (Docs: [Fail2Ban][docs-fail2ban]) -- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behaviour. (Docs: [Fail2Ban][docs-fail2ban]) +- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behavior. (Docs: [Fail2Ban][docs-fail2ban]) - **amavis.cf:** replaces the `/etc/amavis/conf.d/50-user` file - **dovecot.cf:** replaces `/etc/dovecot/local.conf`. (Docs: [Override Dovecot Defaults][docs-override-dovecot]) - **dovecot-quotas.cf:** list of custom quotas per mailbox. (Docs: [Accounts][docs-accounts-quota]) diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index b32288c1..4cb60e77 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -107,7 +107,7 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver The second workaround is slightly more complicated because the `compose.yaml` has to be modified. -As shown in the [fail2ban section](../../security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. +As shown in the [fail2ban section](../security/fail2ban.md#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index d58430e1..2c89948d 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -107,9 +107,9 @@ This could be from outdated software, or running a system that isn't able to pro - **macOS:** DMS has limited support for macOS. Often an issue encountered is due to permissions related to the `volumes` config in `compose.yaml`. You may have luck [trying `gRPC FUSE`][gh-macos-support] as the file sharing implementation; [`VirtioFS` is the successor][docker-macos-virtiofs] but presently appears incompatible with DMS. - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. -- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. +- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behavior of your DMS container, even with the latest Docker Engine installed. - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. -- **Rootless containers:** Introduces additional differences in behaviour or requirements: +- **Rootless containers:** Introduces additional differences in behavior or requirements: - cgroup v2 is required for supporting rootless containers. - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs-rootless-portdriver]. diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 7ae96fcd..1aa6799d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -560,7 +560,7 @@ Changes the interval in which log files are rotated. The Amavis action configured by this setting: - - Influences the behaviour of the [`SA_KILL`](#sa_kill) setting. + - Influences the behavior of the [`SA_KILL`](#sa_kill) setting. - Applies to the Amavis config parameters `$final_spam_destiny` and `$final_bad_header_destiny`. !!! note "This ENV setting is related to" @@ -596,7 +596,7 @@ Mail is not yet considered spam at this spam score, but for purposes like diagno ``` !!! info "The `X-Spam-Score` is `4.162`" - + High enough for `SA_TAG` to trigger adding these headers, but not high enough for `SA_TAG2` (_which would set `X-Spam-Flag: YES` instead_). ##### SA_TAG2 diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index fe9bd5ea..283c2a95 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -22,15 +22,59 @@ The following environment variables are related to Rspamd: 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] 9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) -With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. +With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. + +[docs-spam-to-junk]: ../environment.md#move_spam_to_junk ## The Default Configuration ### Mode of Operation +!!! tip "Attention" + + Read this section carefully if you want to understand how Rspamd is integrated into DMS and how it works (on a surface level). + +Rspamd is integrated as a milter into DMS. When enabled, Postfix's `main.cf` configuration file includes the parameter `rspamd_milter = inet:localhost:11332`, which is added to `smtpd_milters`. As a milter, Rspamd can inspect incoming and outgoing e-mails. + +Each mail is assigned what Rspamd calls symbols: when an e-mail matches a specific criterion, the mail receives a symbol. Afterwards, Rspamd applies a _spam score_ (as usual with anti-spam software) to the e-mail. + +- The score itself is calculated by adding the values of the individual symbols applied earlier. The higher the spam score is, the more likely the e-mail is spam. +- Symbol values can be negative (i.e., these symbols indicate the mail is legitimate, maybe because [SPF and DKIM][docs-dkim-dmarc-spf] are verified successfully) or the symbol can be positive (i.e., these symbols indicate the e-mail is spam, maybe because the e-mail contains a lot of links). + +Rspamd then adds (a few) headers to the e-mail based on the spam score. Most important are `X-Spamd-Result`, which contains an overview of which symbols were applied. It could look like this: + +```txt +X-Spamd-Result default: False [-2.80 / 11.00]; R_SPF_NA(1.50)[no SPF record]; R_DKIM_ALLOW(-1.00)[example.com:s=dtag1]; DWL_DNSWL_LOW(-1.00)[example.com:dkim]; RWL_AMI_LASTHOP(-1.00)[192.0.2.42:from]; DMARC_POLICY_ALLOW(-1.00)[example.com,none]; RWL_MAILSPIKE_EXCELLENT(-0.40)[192.0.2.42:from]; FORGED_SENDER(0.30)[noreply@example.com,some-reply-address@bounce.example.com]; RCVD_IN_DNSWL_LOW(-0.10)[192.0.2.42:from]; MIME_GOOD(-0.10)[multipart/mixed,multipart/related,multipart/alternative,text/plain]; MIME_TRACE(0.00)[0:+,1:+,2:+,3:+,4:~,5:~,6:~]; RCVD_COUNT_THREE(0.00)[3]; RCPT_COUNT_ONE(0.00)[1]; REPLYTO_DN_EQ_FROM_DN(0.00)[]; ARC_NA(0.00)[]; TO_MATCH_ENVRCPT_ALL(0.00)[]; RCVD_TLS_LAST(0.00)[]; DKIM_TRACE(0.00)[example.com:+]; HAS_ATTACHMENT(0.00)[]; TO_DN_NONE(0.00)[]; FROM_NEQ_ENVFROM(0.00)[noreply@example.com,some-reply-address@bounce.example.com]; FROM_HAS_DN(0.00)[]; REPLYTO_DOM_NEQ_FROM_DOM(0.00)[]; PREVIOUSLY_DELIVERED(0.00)[receiver@anotherexample.com]; ASN(0.00)[asn:3320, ipnet:192.0.2.0/24, country:DE]; MID_RHS_MATCH_FROM(0.00)[]; MISSING_XM_UA(0.00)[]; HAS_REPLYTO(0.00)[some-reply-address@dms-reply.example.com] +``` + +And then there is a corresponding `X-Rspamd-Action` header, which shows the overall result and the action that is taken. In our example, it would be: + +```txt +X-Rspamd-Action no action +``` + +Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][rspamd-actions-config] defines what to do at certain scores: + +1. At a score of 4, the e-mail is to be _greylisted_; +2. At a score of 6, the e-mail is _marked with a header_ (`X-Spam: Yes`); +3. At a score of 7, the e-mail will additionally have their _subject re-written_ (appending a prefix like `[SPAM]`); +4. At a score of 11, the e-mail is outright _rejected_. + +--- + +There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][rspamc-docs-bayes] here, nor have we talked about [Sieve rules for e-mails that are marked as spam][docs-spam-to-junk]. + +Even the calculation of the score with the individual symbols has been presented to you in a simplified manner. But with the knowledge from above, you're equipped to read on and use Rspamd confidently. Keep on reading to understand the integration even better - you will want to know about your anti-spam software, not only to keep the bad e-mail out, but also to make sure the good e-mail arrive properly! + +[docs-dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md +[rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/rspamd/local.d/actions.conf +[rspamc-docs-bayes]: https://rspamd.com/doc/configuration/statistic.html + +### Workers + The proxy worker operates in [self-scan mode][rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. @@ -39,9 +83,9 @@ DMS does not set a default password for the controller worker. You may want to d ### Persistence with Redis -When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist it's data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. -Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS rspamd config_). +Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). ### Web Interface @@ -77,13 +121,13 @@ You can find a list of all Rspamd modules [on their website][rspamd-docs-modules #### Disabled By Default -DMS disables certain modules (clickhouse, elastic, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. +DMS disables certain modules (`clickhouse`, `elastic`, `neural`, `reputation`, `spamassassin`, `url_redirector`, `metric_exporter`) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. #### Anti-Virus (ClamAV) You can choose to enable ClamAV, and Rspamd will then use it to check for viruses. Just set the environment variable `ENABLE_CLAMAV=1`. -#### RBLs (Realtime Blacklists) / DNSBLs (DNS-based Blacklists) +#### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to a variety of blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][rbl-vs-dnsbl]\]. @@ -105,12 +149,15 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. +!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs-config-directories]?" + !!! warning "Clashing Overrides" Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories ### With the Help of a Custom File @@ -127,7 +174,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behaviour][rspamd-docs-basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs-basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" diff --git a/docs/content/examples/use-cases/imap-folders.md b/docs/content/examples/use-cases/imap-folders.md index ad5f3ed3..ce90355a 100644 --- a/docs/content/examples/use-cases/imap-folders.md +++ b/docs/content/examples/use-cases/imap-folders.md @@ -61,7 +61,7 @@ Take care to test localized names work well as well. This information is provided by the community. - It presently lacks references to confirm the behaviour. If any information is incorrect please let us know! :smile: + It presently lacks references to confirm the behavior. If any information is incorrect please let us know! :smile: [docs-config-overrides-dovecot]: ../../config/advanced/override-defaults/dovecot.md#override-configuration diff --git a/docs/content/faq.md b/docs/content/faq.md index f666b102..0985c0a9 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -12,7 +12,7 @@ Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is poss ### How are IMAP mailboxes (_aka IMAP Folders_) set up? -`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. +`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](./examples/use-cases/imap-folders.md) for more information. ### How do I update DMS? @@ -128,7 +128,7 @@ find "${PWD}/docker-data/dms-backups/" -type f -mtime +30 -delete ### I Want to Know More About the Ports -See [this part of the documentation](../config/security/understanding-the-ports/) for further details and best practice advice, **especially regarding security concerns**. +See [this part of the documentation](./config/security/understanding-the-ports.md) for further details and best practice advice, **especially regarding security concerns**. ### How can I configure my email client? @@ -210,7 +210,7 @@ baduser@example.com devnull ### What kind of SSL certificates can I use? -Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](../config/environment/#ssl_type) for more details. +Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](./config/environment.md#ssl_type) for more details. ### I just moved from my old mail server to DMS, but "it doesn't work"? diff --git a/docs/content/index.md b/docs/content/index.md index ff1214b1..3739423c 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -24,9 +24,9 @@ If you're completely new to mail servers or you want to read up on them, check o There is also a script - [`setup.sh`][github-file-setupsh] - supplied with this project. It supports you in configuring and administrating your server. Information on how to get it and how to use it is available [on a dedicated page][docs-setupsh]. -[docs-introduction]: ./introduction/ -[docs-usage]: ./usage/ -[docs-examples]: ./examples/tutorials/basic-installation/ +[docs-introduction]: ./introduction.md +[docs-usage]: ./usage.md +[docs-examples]: ./examples/tutorials/basic-installation.md [github-file-setupsh]: https://github.com/docker-mailserver/docker-mailserver/blob/master/setup.sh [docs-setupsh]: ./config/setup.sh/ @@ -59,10 +59,10 @@ You might also want to check out: DMS employs a variety of tests. If you want to know more about our test suite, view our [testing docs][docs-tests]. -[docs-tests]: ./contributing/tests/ +[docs-tests]: ./contributing/tests.md ### Contributing We are always happy to welcome new contributors. For guidelines and entrypoints please have a look at the [Contributing section][docs-contributing]. -[docs-contributing]: ./contributing/issues-and-pull-requests/ +[docs-contributing]: ./contributing/issues-and-pull-requests.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index aaaaf51b..8a6a24b0 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -86,8 +86,8 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.tilde - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: extend_pygments_lang: - name: yml From 2d59aac5a1f3058287e7563884917cbcca9a5c53 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:54:27 +1300 Subject: [PATCH 4/5] chore: Add maintenance comment for `sed` usage (#3789) This is a more explicit reminder for any future contributors that get thrown off by the usage of `sed` here and may be inclined to change it. Add a link to reference a comment where it's already been explored what the alternative `sed` invocations available are. --- target/scripts/startup/setup.d/security/misc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 170f46fb..36d8905f 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -70,6 +70,8 @@ function __setup__security__spamassassin() { if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' + # Maintainers should take care in attempting to change these sed commands. Alternatives were already explored: + # https://github.com/docker-mailserver/docker-mailserver/pull/3767#issuecomment-1885989591 # shellcheck disable=SC2016 sed -i -r 's|^\$sa_tag_level_deflt (.*);|\$sa_tag_level_deflt = '"${SA_TAG}"';|g' /etc/amavis/conf.d/20-debian_defaults From 437114c5dd7928037deea1a2e31535b5a20746ad Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:46:22 +1300 Subject: [PATCH 5/5] tests: Revise `process_check_restart.bats` (#3780) --- CHANGELOG.md | 1 + .../process_check_restart.bats | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b871f0ab..cfdfd314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: + - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. - `swaks` version is now the latest from Github releases instead of the Debian package. diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 4b01454e..0a54a60f 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -11,19 +11,16 @@ function teardown() { _default_teardown ; } # Process matching notes: # opendkim (/usr/sbin/opendkim) - x2 of the same process are found running (1 is the parent) # opendmarc (/usr/sbin/opendmarc) -# master (/usr/lib/postfix/sbin/master) - Postfix main process (Can take a few seconds running to be ready) -# NOTE: pgrep or pkill used with `--full` would also match `/usr/sbin/amavisd-new (master)` -# -# amavi (/usr/sbin/amavi) - Matches three processes, the main process is `/usr/sbin/amavisd-new (master)` -# NOTE: `amavisd-new` can only be matched with `--full`, regardless pkill would return `/usr/sbin/amavi` +# postfix (/usr/lib/postfix/sbin/master) - Postfix main process (two ancestors, launched via pidproxy python3 script) # +# amavisd-new (usr/sbin/amavisd-new) # clamd (/usr/sbin/clamd) # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) -# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - Started by fail2ban-wrapper.sh +# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - NOTE: python3 is due to the shebang # mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) -# postgrey (postgrey) - NOTE: This process lacks path information to match with `--full` in pgrep / pkill -# postsrsd (/usr/sbin/postsrsd) - NOTE: Also matches the wrapper: `/bin/bash /usr/local/bin/postsrsd-wrapper.sh` +# postgrey (postgrey) - NOTE: This process command uses perl via shebang, but unlike python3 the context is missing +# postsrsd (/usr/sbin/postsrsd) # saslauthd (/usr/sbin/saslauthd) - x5 of the same process are found running (1 is a parent of 4) # Delays: @@ -31,17 +28,16 @@ function teardown() { _default_teardown ; } # dovecot + fail2ban, take approx 1 sec to kill properly # opendkim + opendmarc can take up to 6 sec to kill properly # clamd + postsrsd sometimes take 1-3 sec to restart after old process is killed. -# postfix + fail2ban (due to Wrapper scripts) can delay a restart by up to 5 seconds from usage of sleep. # These processes should always be running: CORE_PROCESS_LIST=( - master + postfix ) # These processes can be toggled via ENV: # NOTE: clamd handled in separate test case ENV_PROCESS_LIST=( - amavi + amavisd-new dovecot fail2ban-server fetchmail @@ -89,7 +85,7 @@ ENV_PROCESS_LIST=( done } -# Average time: 23 seconds (29 with wrapper scripts) +# Average time: 23 seconds @test "(enabled ENV) should restart processes when killed" { export CONTAINER_NAME=${CONTAINER2_NAME} local CONTAINER_ARGS_ENV_CUSTOM=( @@ -153,14 +149,18 @@ function _should_restart_when_killed() { # Wait until process has been running for at least MIN_PROCESS_AGE: # (this allows us to more confidently check the process was restarted) _run_until_success_or_timeout 30 _check_if_process_is_running "${PROCESS}" "${MIN_PROCESS_AGE}" - # NOTE: refute_output doesn't have output to compare to when a run failure is due to a timeout + # NOTE: refute_output will not have any output to compare against if a `run` failure is caused by a timeout assert_success assert_output --partial "${PROCESS}" # Should kill the process successfully: # (which should then get restarted by supervisord) - _run_in_container pkill --echo "${PROCESS}" - assert_output --partial "${PROCESS}" + # NOTE: The process name from `pkill --echo` does not always match the equivalent processs name from `pgrep --list-full`. + # The oldest process returned (if multiple) should be the top-level process launched by supervisord, + # the PID will verify the target process was killed correctly: + local PID=$(_exec_in_container pgrep --full --oldest "${PROCESS}") + _run_in_container pkill --echo --full "${PROCESS}" + assert_output --partial "killed (pid ${PID})" assert_success # Wait until original process is not running: @@ -176,13 +176,16 @@ function _should_restart_when_killed() { assert_output --partial "${PROCESS}" } -# NOTE: CONTAINER_NAME is implicit; it should have be set prior to calling. +# NOTE: CONTAINER_NAME is implicit; it should have been set prior to calling. function _check_if_process_is_running() { local PROCESS=${1} local MIN_SECS_RUNNING [[ -n ${2:-} ]] && MIN_SECS_RUNNING=('--older' "${2}") - local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full "${MIN_SECS_RUNNING[@]}" "${PROCESS}") + # `--list-full` provides information for matching against (full process path) + # `--full` allows matching the process against the full path (required if a process is not the exec command, such as started by python3 command without a shebang) + # `--oldest` should select the parent process when there are multiple results, typically the command defined in `supervisor-app.conf` + local IS_RUNNING=$(_exec_in_container pgrep --full --list-full "${MIN_SECS_RUNNING[@]}" --oldest "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]]; then