From 91b2c9834ef5f80f2a6b852ac90d0e5edc99b9a9 Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Tue, 31 Dec 2019 16:34:21 +0100 Subject: [PATCH 01/14] Upgrade to buster and remove filebeat --- Dockerfile | 41 ++++++++++---------- target/filebeat.yml.tmpl | 13 ------- target/supervisor/conf.d/supervisor-app.conf | 8 ---- 3 files changed, 21 insertions(+), 41 deletions(-) delete mode 100644 target/filebeat.yml.tmpl diff --git a/Dockerfile b/Dockerfile index bd8632c0..e6af09b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stretch-slim +FROM debian:buster-slim ARG VCS_REF ARG VCS_VERSION @@ -29,10 +29,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Packages # hadolint ignore=DL3015 -RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /etc/apt/sources.list.d/stretch-bp.list && \ +RUN \ apt-get update -q --fix-missing && \ apt-get -y install postfix && \ - # TODO installing postfix with --no-install-recommends makes "checking ssl: generated default cert works correctly" fail apt-get -y install --no-install-recommends \ amavisd-new \ apt-transport-https \ @@ -77,7 +76,8 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et postsrsd \ pyzor \ razor \ - ripole \ + # TODO not present in buster? + #ripole \ rpm2cpio \ rsyslog \ sasl2-bin \ @@ -88,14 +88,15 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et unzip \ whois \ xz-utils \ - zoo \ - && \ + # TODO not present in buster? + #zoo \ + #&& \ # use Dovecot community repo to react faster on security updates - curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \ - gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \ - echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \ - apt-get update -q --fix-missing && \ - apt-get -y install --no-install-recommends \ + #curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \ + #gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \ + #echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \ + #apt-get update -q --fix-missing && \ + #apt-get -y install --no-install-recommends \ dovecot-core \ dovecot-imapd \ dovecot-ldap \ @@ -117,15 +118,15 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et rm -f /etc/cron.daily/00logwatch # install filebeat for logging -RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ - echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ - apt-get update -q --fix-missing && \ - apt-get -y install --no-install-recommends \ - filebeat \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl +# SKIP and run in an external container instead +#RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ +# echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ +# apt-get update -q --fix-missing && \ +# apt-get -y install --no-install-recommends \ +# filebeat \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* +#COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \ chmod 644 /etc/clamav/freshclam.conf && \ diff --git a/target/filebeat.yml.tmpl b/target/filebeat.yml.tmpl deleted file mode 100644 index d2d84ed4..00000000 --- a/target/filebeat.yml.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -output: - logstash: - enabled: true - hosts: - - $ELK_HOST:$ELK_PORT - -filebeat: - prospectors: - - - paths: - - /var/log/mail/mail.log - document_type: syslog - diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 08b246f5..fd010b4d 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -65,14 +65,6 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/dovecot -F -c /etc/dovecot/dovecot.conf -[program:filebeat] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/bin/filebeat -c /etc/filebeat/filebeat.yml - [program:clamav] startsecs=0 autostart=false From 85ae8a1471d6b522c9949bb644bc05e506f2677b Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Mon, 6 Jan 2020 13:10:46 +0100 Subject: [PATCH 02/14] Fix fail2ban issues and install some suggested amavis packages --- Dockerfile | 10 +++++++++- target/fail2ban/filter.d/postfix-sasl.conf | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 target/fail2ban/filter.d/postfix-sasl.conf diff --git a/Dockerfile b/Dockerfile index e6af09b6..1d4f0a88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ RUN \ apt-get update -q --fix-missing && \ apt-get -y install postfix && \ apt-get -y install --no-install-recommends \ + altermime \ amavisd-new \ apt-transport-https \ arj \ @@ -55,6 +56,7 @@ RUN \ iptables \ locales \ logwatch \ + lhasa \ libdate-manip-perl \ liblz4-tool \ libmail-spf-perl \ @@ -190,7 +192,8 @@ RUN sed -i -r 's/#(@| \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_fil # Configure Fail2ban COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf -RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf && mkdir /var/run/fail2ban +COPY target/fail2ban/filter.d/postfix-sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf +RUN mkdir /var/run/fail2ban # Enables Pyzor and Razor RUN su - amavis -c "razor-admin -create && \ @@ -252,6 +255,11 @@ COPY target/supervisor/conf.d/* /etc/supervisor/conf.d/ WORKDIR / +# Switch iptables and ip6tables to legacy for fail2ban +RUN update-alternatives --set iptables /usr/sbin/iptables-legacy \ + && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy + + EXPOSE 25 587 143 465 993 110 995 4190 CMD ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/target/fail2ban/filter.d/postfix-sasl.conf b/target/fail2ban/filter.d/postfix-sasl.conf new file mode 100644 index 00000000..756b4937 --- /dev/null +++ b/target/fail2ban/filter.d/postfix-sasl.conf @@ -0,0 +1,19 @@ +# Fail2Ban filter for postfix authentication failures + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] + +failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$ + +ignoreregex = authentication failed: Connection lost to authentication server$ + +[Init] + +journalmatch = _SYSTEMD_UNIT=postfix.service + +ignoreregex = From a208748ea2507660923a0728148bec63404e0467 Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Sun, 26 Jan 2020 08:34:40 +0100 Subject: [PATCH 03/14] Configure amavis with D_BOUNCE for spam --- target/amavis/conf.d/49-docker-mailserver | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 target/amavis/conf.d/49-docker-mailserver diff --git a/target/amavis/conf.d/49-docker-mailserver b/target/amavis/conf.d/49-docker-mailserver new file mode 100644 index 00000000..6217d623 --- /dev/null +++ b/target/amavis/conf.d/49-docker-mailserver @@ -0,0 +1,9 @@ +use strict; + +# Override options set in earlier files, use 50-user to override these + +# Bounce spam, the default option for buster is D_PASS to deliver +$final_spam_destiny = D_BOUNCE; + +#------------ Do not modify anything below this line ------------- +1; # ensure a defined return From 42ccae264237c8d2f14b32bf8b7d21c5e528f6a0 Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Sun, 26 Jan 2020 08:36:31 +0100 Subject: [PATCH 04/14] Test stop with a less central container than mail --- test/tests.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.bats b/test/tests.bats index 788bdef5..3b1ebd9e 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -1307,6 +1307,6 @@ EOF # @test "checking that the container stops cleanly" { - run docker stop -t 60 mail + run docker stop -t 60 mail_override_hostname assert_success } From f342151b80529b876a223f5647fb521101694754 Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Sun, 26 Jan 2020 16:39:58 +0100 Subject: [PATCH 05/14] Fixed several amavis tests and removed commented code --- target/amavis/conf.d/49-docker-mailserver | 3 ++ test/tests.bats | 34 ++++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/target/amavis/conf.d/49-docker-mailserver b/target/amavis/conf.d/49-docker-mailserver index 6217d623..3d67ae00 100644 --- a/target/amavis/conf.d/49-docker-mailserver +++ b/target/amavis/conf.d/49-docker-mailserver @@ -5,5 +5,8 @@ use strict; # Bounce spam, the default option for buster is D_PASS to deliver $final_spam_destiny = D_BOUNCE; +# Higher log level to get expected messages at startup +$log_level = 2; + #------------ Do not modify anything below this line ------------- 1; # ensure a defined return diff --git a/test/tests.bats b/test/tests.bats index 3b1ebd9e..8aadb473 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -217,10 +217,8 @@ function count_processed_changes() { } @test "checking smtp: delivers mail to existing account" { - #run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=; 6 ; 1 , orig_to=; 1 , orig_to=; 1 ; 2 ;" cat <<'EOF' | assert_output 1 6 @@ -785,9 +783,37 @@ EOF } @test "checking system: amavis decoders installed and available" { - run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq | tr '\n' ';'" + run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log*|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq" assert_success - assert_output "7z;Z;arc;arj;bz2;cab;cpio;deb;doc;exe;gz;iso;jar;kmz;lha;lrz;lz4;lzma;lzo;mail;rar;rpm;swf;tar;tnef;xz;zip;zoo;" + # Support for doc and zoo removed in buster + cat <<'EOF' | assert_output +7z +Z +arc +arj +bz2 +cab +cpio +deb +exe +gz +iso +jar +kmz +lha +lrz +lz4 +lzma +lzo +mail +rar +rpm +swf +tar +tnef +xz +zip +EOF } From dde6acd1a169f89ddf7dd937c9bddf4e220f4340 Mon Sep 17 00:00:00 2001 From: Erik Wramner Date: Sun, 26 Jan 2020 21:05:24 +0100 Subject: [PATCH 06/14] Search in rotated logs and limit to RelayedInbound --- test/tests.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests.bats b/test/tests.bats index 8aadb473..7d21f509 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -282,7 +282,7 @@ EOF } @test "checking smtp: redirects mail to external aliases" { - run docker exec mail /bin/sh -c "grep -- '-> ' /var/log/mail/mail.log | wc -l" + run docker exec mail /bin/sh -c "grep -- '-> ' /var/log/mail/mail.log* | grep RelayedInbound | wc -l" assert_success assert_output 2 } From d3f7c56cdf084d87cb15726d9182444059e6fbcd Mon Sep 17 00:00:00 2001 From: Robert Pufky Date: Sat, 1 Feb 2020 14:57:03 -0800 Subject: [PATCH 07/14] Fix broken fail2ban dovecot filter; use instead of undocumented feature. * Replace deprecated, undocumented fail2ban feature "(\P\S*)" with supported host match "". * Fixes "No failure-id group in '(?: pop3-login|ima ..." fail2ban dovecot filter error message. * See: https://github.com/fail2ban/fail2ban/issues/2130 --- target/fail2ban/filter.d/dovecot.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/fail2ban/filter.d/dovecot.conf b/target/fail2ban/filter.d/dovecot.conf index 6814b0ca..c83ecef4 100644 --- a/target/fail2ban/filter.d/dovecot.conf +++ b/target/fail2ban/filter.d/dovecot.conf @@ -10,7 +10,7 @@ failregex = ^%(__prefix_line)s(pam_unix(\(dovecot:auth\))?:)?\s+authentication f ^%(__prefix_line)s(pop3|imap)-login: (Info: )?(Aborted login|Disconnected)(: Inactivity)? \(((no auth attempts|auth failed, \d+ attempts)( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<\S*>,)?( method=\S+,)? rip=, lip=(\d{1,3}\.){3}\d{1,3}(, session=<\w+>)?(, TLS( handshaking)?(: Disconnected)?)?\s*$ ^%(__prefix_line)s(Info|dovecot: auth\(default\)): pam\(\S+,\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$ ^\s.*passwd-file\(\S*,\): unknown user.*$ - (?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(?P\S*),.* + (?: pop3-login|imap-login): .*(?:Authentication failure|Aborted login \(auth failed|Aborted login \(tried to use disabled|Disconnected \(auth failed).*rip=(),.* ## ^%(__prefix_line)spasswd-file\(\S*,\): unknown user.*$ -ignoreregex = \ No newline at end of file +ignoreregex = From 30c57b944b51b35896fb78575713057b51f9aa87 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 22 Feb 2020 00:52:32 +0100 Subject: [PATCH 08/14] Allow calling setup.sh from other scripts Calling setup.sh from other scripts fails, for example when adding a new mailbox: the input device is not a TTY --- setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.sh b/setup.sh index 8f94135d..84a5d4fd 100755 --- a/setup.sh +++ b/setup.sh @@ -127,7 +127,7 @@ _docker_image_exists() { fi } -if [ -t 1 ] ; then +if tty -s ; then USE_TTY="-ti" fi From bcb7f402607fb52c9796cfeb6041352ff6b2ce8a Mon Sep 17 00:00:00 2001 From: Rainer Rillke Date: Sun, 1 Mar 2020 19:40:50 +0100 Subject: [PATCH 09/14] README, env file: Warn about open relay configuration Addresses: #1405 --- README.md | 2 +- env-mailserver.dist | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d61e818..04bf351f 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ Enables the Sender Rewriting Scheme. SRS is needed if your mail server acts as f ##### PERMIT_DOCKER -Set different options for mynetworks option (can be overwrite in postfix-main.cf) +Set different options for mynetworks option (can be overwrite in postfix-main.cf) **WARNING**: Adding the docker network's gateway to the list of trusted hosts, e.g. using the `network` or `connected-networks` option, can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay), [for instance](https://github.com/tomav/docker-mailserver/issues/1405#issuecomment-590106498) if IPv6 is enabled on the host machine but not in Docker. - **empty** => localhost only - host => Add docker host (ipv4 only) - network => Add the docker default bridge network (172.16.0.0/12); **WARNING**: `docker-compose` might use others (e.g. 192.168.0.0/16) use `PERMIT_DOCKER=connected-networks` in this case diff --git a/env-mailserver.dist b/env-mailserver.dist index 9556c695..86b8b3cf 100644 --- a/env-mailserver.dist +++ b/env-mailserver.dist @@ -20,6 +20,9 @@ ONE_DIR=0 POSTMASTER_ADDRESS= # Set different options for mynetworks option (can be overwrite in postfix-main.cf) +# **WARNING**: Adding the docker network's gateway to the list of trusted hosts, e.g. using the `network` or +# `connected-networks` option, can create an open relay +# https://github.com/tomav/docker-mailserver/issues/1405#issuecomment-590106498 # empty => localhost only # host => Add docker host (ipv4 only) # network => Add all docker containers (ipv4 only) From d148eeddfb1d7025a2035a9a3a063b0ca2dc6c11 Mon Sep 17 00:00:00 2001 From: Wandrille RONCE Date: Sun, 15 Mar 2020 17:51:12 +0100 Subject: [PATCH 10/14] Add an option to place spam in the inbox, and then sort the mail by a sieve rule for example --- env-mailserver.dist | 3 +++ target/start-mailserver.sh | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/env-mailserver.dist b/env-mailserver.dist index 86b8b3cf..90e3372b 100644 --- a/env-mailserver.dist +++ b/env-mailserver.dist @@ -136,6 +136,9 @@ REPORT_INTERVAL=daily ENABLE_SPAMASSASSIN=0 +#If Enabled, SPAM goes to your inbox with added SPAM header, you can then move it to a specific SPAM folder with SIEVE rules +SPAMASSASSIN_SPAM_TO_INBOX=0 + # add spam info headers if at, or above that level: SA_TAG=2.0 diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index cb3212b2..5b3c04b7 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -37,6 +37,7 @@ DEFAULT_VARS["SRS_SENDER_CLASSES"]="${SRS_SENDER_CLASSES:="envelope_sender"}" DEFAULT_VARS["REPORT_RECIPIENT"]="${REPORT_RECIPIENT:="0"}" DEFAULT_VARS["LOGROTATE_INTERVAL"]="${LOGROTATE_INTERVAL:=${REPORT_INTERVAL:-"daily"}}" DEFAULT_VARS["LOGWATCH_INTERVAL"]="${LOGWATCH_INTERVAL:="none"}" +DEFAULT_VARS["SPAMASSASSIN_SPAM_TO_INBOX"]="${SPAMASSASSIN_SPAM_TO_INBOX:="0"}" DEFAULT_VARS["VIRUSMAILS_DELETE_DELAY"]="${VIRUSMAILS_DELETE_DELAY:="7"}" ########################################################################## @@ -1402,6 +1403,23 @@ function _setup_security_stack() { fi test -e /tmp/docker-mailserver/spamassassin-rules.cf && cp /tmp/docker-mailserver/spamassassin-rules.cf /etc/spamassassin/ + + + if [ "$SPAMASSASSIN_SPAM_TO_INBOX" = "1" ]; then + notify 'inf' "Configure Spamassassin/Amavis to put SPAM inbox" + bannedbouncecheck=`egrep "final_banned_destiny.*D_BOUNCE" /etc/amavis/conf.d/20-debian_defaults` + if [ -n "$bannedbouncecheck" ] ; + then + sed -i "/final_banned_destiny/ s|D_BOUNCE|D_REJECT|" /etc/amavis/conf.d/20-debian_defaults + fi + + finalbouncecheck=`egrep "final_spam_destiny.*D_BOUNCE" /etc/amavis/conf.d/20-debian_defaults` + if [ -n "$finalbouncecheck" ] ; + then + sed -i "/final_spam_destiny/ s|D_BOUNCE|D_PASS|" /etc/amavis/conf.d/20-debian_defaults + fi + fi + fi # Clamav From 2a7e3b861f448875b97f5197fb57f8d6e09387d8 Mon Sep 17 00:00:00 2001 From: Germain Masse Date: Thu, 19 Mar 2020 15:22:31 +0100 Subject: [PATCH 11/14] Support GeoIP license number in ELK Dockerfile --- docker-compose.elk.yml.dist | 7 ++++++- elk/.env.dist | 1 + elk/Dockerfile | 9 ++++++--- elk/docker-compose.yml.dist | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 elk/.env.dist create mode 100644 elk/docker-compose.yml.dist diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist index 0c308a3c..9fb1ff71 100644 --- a/docker-compose.elk.yml.dist +++ b/docker-compose.elk.yml.dist @@ -25,12 +25,17 @@ services: - SYS_PTRACE restart: always elk: - build: elk + build: + context: elk + args: + - MAXMIND_LICENSE ports: - "5601:5601" - "9200:9200" - "5044:5044" - "5000:5000" + env_file: + - elk/.env restart: always volumes: diff --git a/elk/.env.dist b/elk/.env.dist new file mode 100644 index 00000000..43a62e16 --- /dev/null +++ b/elk/.env.dist @@ -0,0 +1 @@ +MAXMIND_LICENSE= diff --git a/elk/Dockerfile b/elk/Dockerfile index de97d26f..6cfa3b1d 100644 --- a/elk/Dockerfile +++ b/elk/Dockerfile @@ -10,15 +10,18 @@ COPY 16-amavis.conf /etc/logstash/conf.d # dovecot grok and filter RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf -# FIXME: may be a cron job? +# FIXME: may be a cron job? SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG MAXMIND_LICENSE RUN mkdir -p /usr/share/GeoIP && \ - curl -L http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat +curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \ +| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat WORKDIR ${LOGSTASH_HOME} RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip -# override beats input +# override beats input COPY 02-beats-input.conf /etc/logstash/conf.d/ # override syslog COPY 10-syslog.conf /etc/logstash/conf.d/ diff --git a/elk/docker-compose.yml.dist b/elk/docker-compose.yml.dist new file mode 100644 index 00000000..1bdfbb25 --- /dev/null +++ b/elk/docker-compose.yml.dist @@ -0,0 +1,16 @@ +version: '2' + +services: + elk: + build: + context: . + args: + - MAXMIND_LICENSE + ports: + - "5601:5601" + - "9200:9200" + - "5044:5044" + - "5000:5000" + env_file: + - .env + restart: always From 03a095ea4d877eb649fd2757c85a1639dd0b1de7 Mon Sep 17 00:00:00 2001 From: Germain Masse Date: Thu, 19 Mar 2020 15:24:26 +0100 Subject: [PATCH 12/14] Bump to ELK 7.6.1 --- elk/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elk/Dockerfile b/elk/Dockerfile index 6cfa3b1d..98a35316 100644 --- a/elk/Dockerfile +++ b/elk/Dockerfile @@ -1,4 +1,4 @@ -FROM sebp/elk:720 +FROM sebp/elk:761 RUN mkdir /etc/logstash/patterns.d #postfix grok and filter From ce41f608884b2f82cc700cfa14e4f8d1065ff153 Mon Sep 17 00:00:00 2001 From: Germain Masse Date: Thu, 19 Mar 2020 16:04:21 +0100 Subject: [PATCH 13/14] Move filebeat to its own container --- Dockerfile | 11 --------- README.md | 2 +- config/filebeat.docker.yml | 16 ++++++++++++ docker-compose.elk.yml.dist | 13 ++++++++++ docker-compose.filebeat.yml.dist | 42 ++++++++++++++++++++++++++++++++ target/start-mailserver.sh | 25 ------------------- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 config/filebeat.docker.yml create mode 100644 docker-compose.filebeat.yml.dist diff --git a/Dockerfile b/Dockerfile index 1d4f0a88..f6d0ba6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,17 +119,6 @@ RUN \ rm -f /etc/postsrsd.secret && \ rm -f /etc/cron.daily/00logwatch -# install filebeat for logging -# SKIP and run in an external container instead -#RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ -# echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ -# apt-get update -q --fix-missing && \ -# apt-get -y install --no-install-recommends \ -# filebeat \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* -#COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl - RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \ chmod 644 /etc/clamav/freshclam.conf && \ freshclam && \ diff --git a/README.md b/README.md index 04bf351f..5d304172 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh) -`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. +`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. #### Exposed ports * 25 receiving email from other mailservers diff --git a/config/filebeat.docker.yml b/config/filebeat.docker.yml new file mode 100644 index 00000000..cfa132ee --- /dev/null +++ b/config/filebeat.docker.yml @@ -0,0 +1,16 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + hints.default_config.enabled: false + +processors: +- add_cloud_metadata: ~ + +output.logstash: + hosts: ["127.0.0.1:5044"] diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist index 0c308a3c..95f3aa2f 100644 --- a/docker-compose.elk.yml.dist +++ b/docker-compose.elk.yml.dist @@ -8,6 +8,10 @@ services: container_name: ${CONTAINER_NAME} links: - elk + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" ports: - "25:25" - "143:143" @@ -24,6 +28,15 @@ services: - NET_ADMIN - SYS_PTRACE restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always elk: build: elk ports: diff --git a/docker-compose.filebeat.yml.dist b/docker-compose.filebeat.yml.dist new file mode 100644 index 00000000..5dc483c1 --- /dev/null +++ b/docker-compose.filebeat.yml.dist @@ -0,0 +1,42 @@ +version: '2' +services: + mail: + image: tvial/docker-mailserver:latest + hostname: ${HOSTNAME} + domainname: ${DOMAINNAME} + container_name: ${CONTAINER_NAME} + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" + volumes: + - maildata:/var/mail + - mailstate:/var/mail-state + - maillogs:/var/log/mail + - ./config/:/tmp/docker-mailserver/ + env_file: + - .env + - env-mailserver + cap_add: + - NET_ADMIN + - SYS_PTRACE + restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always +volumes: + maildata: + driver: local + maillogs: + driver: local diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 5b3c04b7..dc31e3cc 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -95,10 +95,6 @@ function register_functions() { _register_setup_function "_setup_default_vars" _register_setup_function "_setup_file_permissions" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_setup_function "_setup_elk_forwarder" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_setup_function "_setup_dovecot" _register_setup_function "_setup_dovecot_dhparam" @@ -208,10 +204,6 @@ function register_functions() { _register_start_daemon "_start_daemons_cron" _register_start_daemon "_start_daemons_rsyslog" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_start_daemon "_start_daemons_filebeat" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_start_daemon "_start_daemons_dovecot" fi @@ -1452,18 +1444,6 @@ function _setup_security_stack() { fi } -function _setup_elk_forwarder() { - notify 'task' 'Setting up Elk forwarder' - - ELK_PORT=${ELK_PORT:="5044"} - ELK_HOST=${ELK_HOST:="elk"} - notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" - cat /etc/filebeat/filebeat.yml.tmpl \ - | sed "s@\$ELK_HOST@$ELK_HOST@g" \ - | sed "s@\$ELK_PORT@$ELK_PORT@g" \ - > /etc/filebeat/filebeat.yml -} - function _setup_logrotate() { notify 'inf' "Setting up logrotate" @@ -1750,11 +1730,6 @@ function _start_daemons_dovecot() { #fi } -function _start_daemons_filebeat() { - notify 'task' 'Starting filebeat' 'n' - supervisorctl start filebeat -} - function _start_daemons_fetchmail() { notify 'task' 'Starting fetchmail' 'n' /usr/local/bin/setup-fetchmail From a00dced8bcabd728bf8f4709700df0d6d8ddda68 Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Thu, 19 Mar 2020 08:35:25 +0000 Subject: [PATCH 14/14] Allow to set comfortably inet_protocols Setting `inet_protocols = ipv4` is almost a requirement when running behind Docker. Provide a way to make it easy. @Tecnativa TT22925 --- README.md | 10 ++++++++++ env-mailserver.dist | 8 ++++++++ target/start-mailserver.sh | 11 ++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d304172..c7530aed 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,8 @@ Set different options for mynetworks option (can be overwrite in postfix-main.cf - network => Add the docker default bridge network (172.16.0.0/12); **WARNING**: `docker-compose` might use others (e.g. 192.168.0.0/16) use `PERMIT_DOCKER=connected-networks` in this case - connected-networks => Add all connected docker networks (ipv4 only) +Note: you probably want to [set `POSTFIX_INET_PROTOCOLS=ipv4`](#postfix_inet_protocols) to make it work fine with Docker. + ##### VIRUSMAILS_DELETE_DELAY Set how many days a virusmail will stay on the server before being deleted @@ -369,6 +371,14 @@ Set the message size limit for all users. If set to zero, the size will be unlim This option has been added in November 2019. Using other format than Maildir is considered as experimental in docker-mailserver and should only be used for testing purpose. For more details, please refer to [Dovecot Documentation](https://wiki2.dovecot.org/MailboxFormat). +##### POSTFIX_INET_PROTOCOLS + +- **all** => All possible protocols. +- ipv4 => Use only IPv4 traffic. Most likely you want this behind Docker. +- ipv6 => Use only IPv6 traffic. + +Note: More details in http://www.postfix.org/postconf.5.html#inet_protocols + ## Reports ##### PFLOGSUMM_TRIGGER diff --git a/env-mailserver.dist b/env-mailserver.dist index 90e3372b..2113998e 100644 --- a/env-mailserver.dist +++ b/env-mailserver.dist @@ -130,6 +130,14 @@ REPORT_SENDER= # Note: This Variable actually controls logrotate inside the container and rotates the log depending on this setting. The main log output is still available in its entirety via `docker logs mail` (Or your respective container name). If you want to control logrotation for the docker generated logfile see: [Docker Logging Drivers](https://docs.docker.com/config/containers/logging/configure/) REPORT_INTERVAL=daily +# Choose TCP/IP protocols to use +# **all** => All possible protocols. +# ipv4 => Use only IPv4 traffic. Most likely you want this behind Docker. +# ipv6 => Use only IPv6 traffic. +# +# Note: More details in http://www.postfix.org/postconf.5.html#inet_protocols +POSTFIX_INET_PROTOCOLS=all + # ----------------------------------------------------------------------------------------------------------------------------- # --------------------- Spamassassin section ---------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------- diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index dc31e3cc..4edbe4a8 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -25,6 +25,7 @@ DEFAULT_VARS["POSTGREY_AUTO_WHITELIST_CLIENTS"]="${POSTGREY_AUTO_WHITELIST_CLIEN DEFAULT_VARS["POSTGREY_TEXT"]="${POSTGREY_TEXT:="Delayed by postgrey"}" DEFAULT_VARS["POSTFIX_MESSAGE_SIZE_LIMIT"]="${POSTFIX_MESSAGE_SIZE_LIMIT:="10240000"}" # ~10 MB by default DEFAULT_VARS["POSTFIX_MAILBOX_SIZE_LIMIT"]="${POSTFIX_MAILBOX_SIZE_LIMIT:="0"}" # no limit by default +DEFAULT_VARS["POSTFIX_INET_PROTOCOLS"]="${POSTFIX_INET_PROTOCOLS:="all"}" DEFAULT_VARS["ENABLE_SASLAUTHD"]="${ENABLE_SASLAUTHD:="0"}" DEFAULT_VARS["SMTP_ONLY"]="${SMTP_ONLY:="0"}" DEFAULT_VARS["DMS_DEBUG"]="${DMS_DEBUG:="0"}" @@ -115,6 +116,9 @@ function register_functions() { _register_setup_function "_setup_dkim" _register_setup_function "_setup_ssl" + if [ "$POSTFIX_INET_PROTOCOLS" != "all" ]; then + _register_setup_function "_setup_inet_protocols" + fi _register_setup_function "_setup_docker_permit" _register_setup_function "_setup_mailname" @@ -172,7 +176,7 @@ function register_functions() { if [ "$LOGWATCH_TRIGGER" != "none" ]; then _register_setup_function "_setup_logwatch" fi - + _register_setup_function "_setup_user_patches" # Compute last as the config files are modified in-place @@ -1093,6 +1097,11 @@ function _setup_postfix_vhost() { fi } +function _setup_inet_protocols() { + notify 'task' 'Setting up POSTFIX_INET_PROTOCOLS option' + postconf -e "inet_protocols = $POSTFIX_INET_PROTOCOLS" +} + function _setup_docker_permit() { notify 'task' 'Setting up PERMIT_DOCKER Option'