Merge pull request #1447 from erik-wramner/next_to_master

Merge next branch with Debian Buster into master.
2024-01-19 02:48:50 +00:00 · 2020-04-05 11:33:51 +02:00 · 2020-04-05 11:33:51 +02:00 · e81704483e
parent 04777fdb89 73b8d65dd3
commit e81704483e
14 changed files with 188 additions and 83 deletions
--- a/44
+++ b/44
@ -1,4 +1,4 @@
-FROM debian:stretch-slim
+FROM debian:buster-slim

 ARG VCS_REF
 ARG VCS_VERSION
@ -29,11 +29,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]

 # Packages
 # hadolint ignore=DL3015
-RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /etc/apt/sources.list.d/stretch-bp.list && \
+RUN \
  apt-get update -q --fix-missing && \
  apt-get -y install postfix && \
-  # TODO installing postfix with --no-install-recommends makes "checking ssl: generated default cert works correctly" fail
  apt-get -y install --no-install-recommends \
+    altermime \
    amavisd-new \
    apt-transport-https \
    arj \
@ -56,6 +56,7 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
    iptables \
    locales \
    logwatch \
+    lhasa \
    libdate-manip-perl \
    liblz4-tool \
    libmail-spf-perl \
@ -77,7 +78,8 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
    postsrsd \
    pyzor \
    razor \
-    ripole \
+    # TODO not present in buster?
+    #ripole \
    rpm2cpio \
    rsyslog \
    sasl2-bin \
@ -88,14 +90,15 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
    unzip \
    whois \
    xz-utils \
-    zoo \
-    && \
+    # TODO not present in buster?
+    #zoo \
+    #&& \
  # use Dovecot community repo to react faster on security updates
-  curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \
-  gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \
-  echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \
-  apt-get update -q --fix-missing && \
-  apt-get -y install --no-install-recommends \
+  #curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \
+  #gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \
+  #echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \
+  #apt-get update -q --fix-missing && \
+  #apt-get -y install --no-install-recommends \
    dovecot-core \
    dovecot-imapd \
    dovecot-ldap \
@ -116,17 +119,6 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et
  rm -f /etc/postsrsd.secret && \
  rm -f /etc/cron.daily/00logwatch

-# install filebeat for logging
-RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \
-  echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \
-  apt-get update -q --fix-missing && \
-  apt-get -y install --no-install-recommends \
-    filebeat \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
-
 RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \
  chmod 644 /etc/clamav/freshclam.conf && \
  freshclam && \
@ -189,7 +181,8 @@ RUN sed -i -r 's/#(@|   \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_fil
 # Configure Fail2ban
 COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf
 COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf
-RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf && mkdir /var/run/fail2ban
+COPY target/fail2ban/filter.d/postfix-sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf
+RUN mkdir /var/run/fail2ban

 # Enables Pyzor and Razor
 RUN su - amavis -c "razor-admin -create && \
@ -251,6 +244,11 @@ COPY target/supervisor/conf.d/* /etc/supervisor/conf.d/

 WORKDIR /

+# Switch iptables and ip6tables to legacy for fail2ban
+RUN update-alternatives --set iptables /usr/sbin/iptables-legacy \
+ && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+
+
 EXPOSE 25 587 143 465 993 110 995 4190

 CMD ["supervisord", "-c", "/etc/supervisor/supervisord.conf"]
--- a/README.md
+++ b/README.md
@ -9,8 +9,8 @@ Easy to deploy and upgrade.

 ## ANNOUNCEMENT

-Some time VERY SOON we will merge the next branch based on Debian Buster to master.
-That means the docker image latest will change. The change may break things!
+At this point we have merged the next branch based on Debian Buster into master.
+That means the docker image latest uses Buster. The change may break things!

 The following possibly breaking changes are known:
 - Filebeat is removed and should be handled by another container, see [Wiki](https://github.com/tomav/docker-mailserver/wiki/).
@ -122,7 +122,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git

 Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh)

-`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.
+`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit.

 #### Exposed ports
 * 25 receiving email from other mailservers
--- a/config/filebeat.docker.yml
+++ b/config/filebeat.docker.yml
@ -0,0 +1,16 @@
+filebeat.config:
+  modules:
+    path: ${path.config}/modules.d/*.yml
+    reload.enabled: false
+
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      hints.default_config.enabled: false
+
+processors:
+- add_cloud_metadata: ~
+
+output.logstash:
+    hosts: ["127.0.0.1:5044"]
--- a/docker-compose.elk.yml.dist
+++ b/docker-compose.elk.yml.dist
@ -8,6 +8,10 @@ services:
    container_name: ${CONTAINER_NAME}
    links:
    - elk
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
    ports:
    - "25:25"
    - "143:143"
@ -24,13 +28,27 @@ services:
    - NET_ADMIN
    - SYS_PTRACE
    restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
  elk:
-     build: elk
+     build:
+       context: elk
+       args:
+       - MAXMIND_LICENSE
     ports:
     - "5601:5601"
     - "9200:9200"
     - "5044:5044"
     - "5000:5000"
+     env_file:
+     - elk/.env
     restart: always

 volumes:
--- a/docker-compose.filebeat.yml.dist
+++ b/docker-compose.filebeat.yml.dist
@ -0,0 +1,42 @@
+version: '2'
+services:
+  mail:
+    image: tvial/docker-mailserver:latest
+    hostname: ${HOSTNAME}
+    domainname: ${DOMAINNAME}
+    container_name: ${CONTAINER_NAME}
+    ports:
+    - "25:25"
+    - "143:143"
+    - "587:587"
+    - "993:993"
+    labels:
+    - "co.elastic.logs/enabled=true"
+    - "co.elastic.logs/module=system"
+    - "co.elastic.logs/fileset.stdout=syslog"
+    volumes:
+    - maildata:/var/mail
+    - mailstate:/var/mail-state
+    - maillogs:/var/log/mail
+    - ./config/:/tmp/docker-mailserver/
+    env_file:
+    - .env
+    - env-mailserver
+    cap_add:
+    - NET_ADMIN
+    - SYS_PTRACE
+    restart: always
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:7.6.1
+    user: root
+    volumes:
+    - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
+    - /var/run/docker.sock:/var/run/docker.sock:ro
+    - /var/lib/docker/containers/:/var/lib/docker/containers/:ro
+    command: ["filebeat", "-e", "--strict.perms=false"]
+    restart: always
+volumes:
+  maildata:
+    driver: local
+  maillogs:
+    driver: local
--- a/elk/.env.dist
+++ b/elk/.env.dist
@ -0,0 +1 @@
+MAXMIND_LICENSE=
--- a/elk/Dockerfile
+++ b/elk/Dockerfile
@ -1,4 +1,4 @@
-FROM sebp/elk:720
+FROM sebp/elk:761

 RUN mkdir /etc/logstash/patterns.d
 #postfix grok and filter
@ -10,15 +10,18 @@ COPY 16-amavis.conf /etc/logstash/conf.d
 # dovecot grok and filter
 RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
 RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
-# FIXME: may be a cron job? 
+# FIXME: may be a cron job?
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+ARG MAXMIND_LICENSE
 RUN mkdir  -p /usr/share/GeoIP && \
- curl -L http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat 
+curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \
+| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat

 WORKDIR ${LOGSTASH_HOME}
 RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip

-# override beats input 
+# override beats input
 COPY 02-beats-input.conf /etc/logstash/conf.d/
 # override syslog
 COPY 10-syslog.conf /etc/logstash/conf.d/
--- a/elk/docker-compose.yml.dist
+++ b/elk/docker-compose.yml.dist
@ -0,0 +1,16 @@
+version: '2'
+
+services:
+  elk:
+     build:
+       context: .
+       args:
+       - MAXMIND_LICENSE
+     ports:
+     - "5601:5601"
+     - "9200:9200"
+     - "5044:5044"
+     - "5000:5000"
+     env_file:
+     - .env
+     restart: always
--- a/target/amavis/conf.d/49-docker-mailserver
+++ b/target/amavis/conf.d/49-docker-mailserver
@ -0,0 +1,12 @@
+use strict;
+
+# Override options set in earlier files, use 50-user to override these
+
+# Bounce spam, the default option for buster is D_PASS to deliver
+$final_spam_destiny = D_BOUNCE;
+
+# Higher log level to get expected messages at startup
+$log_level = 2;
+
+#------------ Do not modify anything below this line -------------
+1;  # ensure a defined return
--- a/target/fail2ban/filter.d/postfix-sasl.conf
+++ b/target/fail2ban/filter.d/postfix-sasl.conf
@ -0,0 +1,19 @@
+# Fail2Ban filter for postfix authentication failures
+
+[INCLUDES]
+
+before = common.conf
+
+[Definition]
+
+_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds]
+
+failregex = ^%(__prefix_line)swarning: [-._\w]+\[<HOST>\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$
+
+ignoreregex = authentication failed: Connection lost to authentication server$
+
+[Init]
+
+journalmatch = _SYSTEMD_UNIT=postfix.service
+
+ignoreregex =
--- a/target/filebeat.yml.tmpl
+++ b/target/filebeat.yml.tmpl
@ -1,13 +0,0 @@
-output:
-  logstash:
-    enabled: true
-    hosts:
-      - $ELK_HOST:$ELK_PORT
-
-filebeat:
-  prospectors:
-    -
-      paths:
-        - /var/log/mail/mail.log
-      document_type: syslog
-
--- a/target/start-mailserver.sh
+++ b/target/start-mailserver.sh
@ -96,10 +96,6 @@ function register_functions() {
 	_register_setup_function "_setup_default_vars"
 	_register_setup_function "_setup_file_permissions"

-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_setup_function "_setup_elk_forwarder"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_setup_function "_setup_dovecot"
                _register_setup_function "_setup_dovecot_dhparam"
@ -212,10 +208,6 @@ function register_functions() {
 	_register_start_daemon "_start_daemons_cron"
 	_register_start_daemon "_start_daemons_rsyslog"

-	if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then
-		_register_start_daemon "_start_daemons_filebeat"
-	fi
-
 	if [ "$SMTP_ONLY" != 1 ]; then
 		_register_start_daemon "_start_daemons_dovecot"
 	fi
@ -1465,18 +1457,6 @@ function _setup_security_stack() {
 	fi
 }

-function _setup_elk_forwarder() {
-	notify 'task' 'Setting up Elk forwarder'
-
-	ELK_PORT=${ELK_PORT:="5044"}
-	ELK_HOST=${ELK_HOST:="elk"}
-	notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)"
-	cat /etc/filebeat/filebeat.yml.tmpl \
-		| sed "s@\$ELK_HOST@$ELK_HOST@g" \
-		| sed "s@\$ELK_PORT@$ELK_PORT@g" \
-		> /etc/filebeat/filebeat.yml
-}
-
 function _setup_logrotate() {
 	notify 'inf' "Setting up logrotate"

@ -1763,11 +1743,6 @@ function _start_daemons_dovecot() {
 	#fi
 }

-function _start_daemons_filebeat() {
-	notify 'task' 'Starting filebeat' 'n'
-    supervisorctl start filebeat
-}
-
 function _start_daemons_fetchmail() {
 	notify 'task' 'Starting fetchmail' 'n'
 	/usr/local/bin/setup-fetchmail
--- a/target/supervisor/conf.d/supervisor-app.conf
+++ b/target/supervisor/conf.d/supervisor-app.conf
@ -65,14 +65,6 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log
 stderr_logfile=/var/log/supervisor/%(program_name)s.log
 command=/usr/sbin/dovecot -F -c /etc/dovecot/dovecot.conf

-[program:filebeat]
-startsecs=0
-autostart=false
-autorestart=true
-stdout_logfile=/var/log/supervisor/%(program_name)s.log
-stderr_logfile=/var/log/supervisor/%(program_name)s.log
-command=/usr/bin/filebeat -c /etc/filebeat/filebeat.yml
-
 [program:clamav]
 startsecs=0
 autostart=false
--- a/test/tests.bats
+++ b/test/tests.bats
@ -217,10 +217,8 @@ function count_processed_changes() {
 }

@test "checking smtp: delivers mail to existing account" {
-  #run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=</</g' | sed 's/, relay.*//g' | sort | uniq -c | tr -s \" \" | tr '\n' ';'"
  run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=</</g' | sed 's/, relay.*//g' | sort | uniq -c | tr -s \" \""
  assert_success
-  #assert_output " 1 <added@localhost.localdomain>; 6 <user1@localhost.localdomain>; 1 <user1@localhost.localdomain>, orig_to=<postmaster@my-domain.com>; 1 <user1@localhost.localdomain>, orig_to=<root>; 1 <user1~test@localhost.localdomain>; 2 <user2@otherdomain.tld>;"
  cat <<'EOF' | assert_output
 1 <added@localhost.localdomain>
 6 <user1@localhost.localdomain>
@ -284,7 +282,7 @@ EOF
 }

@test "checking smtp: redirects mail to external aliases" {
-  run docker exec mail /bin/sh -c "grep -- '-> <external1@otherdomain.tld>' /var/log/mail/mail.log | wc -l"
+  run docker exec mail /bin/sh -c "grep -- '-> <external1@otherdomain.tld>' /var/log/mail/mail.log* | grep RelayedInbound | wc -l"
  assert_success
  assert_output 2
 }
@ -785,9 +783,37 @@ EOF
 }

@test "checking system: amavis decoders installed and available" {
-  run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq | tr '\n' ';'"
+  run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log*|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq"
  assert_success
-  assert_output "7z;Z;arc;arj;bz2;cab;cpio;deb;doc;exe;gz;iso;jar;kmz;lha;lrz;lz4;lzma;lzo;mail;rar;rpm;swf;tar;tnef;xz;zip;zoo;"
+  # Support for doc and zoo removed in buster
+  cat <<'EOF' | assert_output
+7z
+Z
+arc
+arj
+bz2
+cab
+cpio
+deb
+exe
+gz
+iso
+jar
+kmz
+lha
+lrz
+lz4
+lzma
+lzo
+mail
+rar
+rpm
+swf
+tar
+tnef
+xz
+zip
+EOF
 }


@ -1307,6 +1333,6 @@ EOF
 #

@test "checking that the container stops cleanly" {
-  run docker stop -t 60 mail
+  run docker stop -t 60 mail_override_hostname
  assert_success
 }