diff --git a/Dockerfile b/Dockerfile index bd8632c0..f6d0ba6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:stretch-slim +FROM debian:buster-slim ARG VCS_REF ARG VCS_VERSION @@ -29,11 +29,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Packages # hadolint ignore=DL3015 -RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /etc/apt/sources.list.d/stretch-bp.list && \ +RUN \ apt-get update -q --fix-missing && \ apt-get -y install postfix && \ - # TODO installing postfix with --no-install-recommends makes "checking ssl: generated default cert works correctly" fail apt-get -y install --no-install-recommends \ + altermime \ amavisd-new \ apt-transport-https \ arj \ @@ -56,6 +56,7 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et iptables \ locales \ logwatch \ + lhasa \ libdate-manip-perl \ liblz4-tool \ libmail-spf-perl \ @@ -77,7 +78,8 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et postsrsd \ pyzor \ razor \ - ripole \ + # TODO not present in buster? + #ripole \ rpm2cpio \ rsyslog \ sasl2-bin \ @@ -88,14 +90,15 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et unzip \ whois \ xz-utils \ - zoo \ - && \ + # TODO not present in buster? + #zoo \ + #&& \ # use Dovecot community repo to react faster on security updates - curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \ - gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \ - echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \ - apt-get update -q --fix-missing && \ - apt-get -y install --no-install-recommends \ + #curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import && \ + #gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg && \ + #echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/stretch stretch main" > /etc/apt/sources.list.d/dovecot-community.list && \ + #apt-get update -q --fix-missing && \ + #apt-get -y install --no-install-recommends \ dovecot-core \ dovecot-imapd \ dovecot-ldap \ @@ -116,17 +119,6 @@ RUN echo "deb http://http.debian.net/debian stretch-backports main" | tee -a /et rm -f /etc/postsrsd.secret && \ rm -f /etc/cron.daily/00logwatch -# install filebeat for logging -RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ - echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ - apt-get update -q --fix-missing && \ - apt-get -y install --no-install-recommends \ - filebeat \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl - RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \ chmod 644 /etc/clamav/freshclam.conf && \ freshclam && \ @@ -189,7 +181,8 @@ RUN sed -i -r 's/#(@| \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_fil # Configure Fail2ban COPY target/fail2ban/jail.conf /etc/fail2ban/jail.conf COPY target/fail2ban/filter.d/dovecot.conf /etc/fail2ban/filter.d/dovecot.conf -RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf && mkdir /var/run/fail2ban +COPY target/fail2ban/filter.d/postfix-sasl.conf /etc/fail2ban/filter.d/postfix-sasl.conf +RUN mkdir /var/run/fail2ban # Enables Pyzor and Razor RUN su - amavis -c "razor-admin -create && \ @@ -251,6 +244,11 @@ COPY target/supervisor/conf.d/* /etc/supervisor/conf.d/ WORKDIR / +# Switch iptables and ip6tables to legacy for fail2ban +RUN update-alternatives --set iptables /usr/sbin/iptables-legacy \ + && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy + + EXPOSE 25 587 143 465 993 110 995 4190 CMD ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/README.md b/README.md index 56d66652..ed4c8712 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ Easy to deploy and upgrade. ## ANNOUNCEMENT -Some time VERY SOON we will merge the next branch based on Debian Buster to master. -That means the docker image latest will change. The change may break things! +At this point we have merged the next branch based on Debian Buster into master. +That means the docker image latest uses Buster. The change may break things! The following possibly breaking changes are known: - Filebeat is removed and should be handled by another container, see [Wiki](https://github.com/tomav/docker-mailserver/wiki/). @@ -122,7 +122,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh) -`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. +`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. #### Exposed ports * 25 receiving email from other mailservers diff --git a/config/filebeat.docker.yml b/config/filebeat.docker.yml new file mode 100644 index 00000000..cfa132ee --- /dev/null +++ b/config/filebeat.docker.yml @@ -0,0 +1,16 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + hints.default_config.enabled: false + +processors: +- add_cloud_metadata: ~ + +output.logstash: + hosts: ["127.0.0.1:5044"] diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist index 0c308a3c..2621b577 100644 --- a/docker-compose.elk.yml.dist +++ b/docker-compose.elk.yml.dist @@ -8,6 +8,10 @@ services: container_name: ${CONTAINER_NAME} links: - elk + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" ports: - "25:25" - "143:143" @@ -24,13 +28,27 @@ services: - NET_ADMIN - SYS_PTRACE restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always elk: - build: elk + build: + context: elk + args: + - MAXMIND_LICENSE ports: - "5601:5601" - "9200:9200" - "5044:5044" - "5000:5000" + env_file: + - elk/.env restart: always volumes: diff --git a/docker-compose.filebeat.yml.dist b/docker-compose.filebeat.yml.dist new file mode 100644 index 00000000..5dc483c1 --- /dev/null +++ b/docker-compose.filebeat.yml.dist @@ -0,0 +1,42 @@ +version: '2' +services: + mail: + image: tvial/docker-mailserver:latest + hostname: ${HOSTNAME} + domainname: ${DOMAINNAME} + container_name: ${CONTAINER_NAME} + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" + volumes: + - maildata:/var/mail + - mailstate:/var/mail-state + - maillogs:/var/log/mail + - ./config/:/tmp/docker-mailserver/ + env_file: + - .env + - env-mailserver + cap_add: + - NET_ADMIN + - SYS_PTRACE + restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always +volumes: + maildata: + driver: local + maillogs: + driver: local diff --git a/elk/.env.dist b/elk/.env.dist new file mode 100644 index 00000000..43a62e16 --- /dev/null +++ b/elk/.env.dist @@ -0,0 +1 @@ +MAXMIND_LICENSE= diff --git a/elk/Dockerfile b/elk/Dockerfile index de97d26f..98a35316 100644 --- a/elk/Dockerfile +++ b/elk/Dockerfile @@ -1,4 +1,4 @@ -FROM sebp/elk:720 +FROM sebp/elk:761 RUN mkdir /etc/logstash/patterns.d #postfix grok and filter @@ -10,15 +10,18 @@ COPY 16-amavis.conf /etc/logstash/conf.d # dovecot grok and filter RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf -# FIXME: may be a cron job? +# FIXME: may be a cron job? SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG MAXMIND_LICENSE RUN mkdir -p /usr/share/GeoIP && \ - curl -L http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz | gunzip -c - > /usr/share/GeoIP/GeoLiteCity.dat +curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \ +| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat WORKDIR ${LOGSTASH_HOME} RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip -# override beats input +# override beats input COPY 02-beats-input.conf /etc/logstash/conf.d/ # override syslog COPY 10-syslog.conf /etc/logstash/conf.d/ diff --git a/elk/docker-compose.yml.dist b/elk/docker-compose.yml.dist new file mode 100644 index 00000000..1bdfbb25 --- /dev/null +++ b/elk/docker-compose.yml.dist @@ -0,0 +1,16 @@ +version: '2' + +services: + elk: + build: + context: . + args: + - MAXMIND_LICENSE + ports: + - "5601:5601" + - "9200:9200" + - "5044:5044" + - "5000:5000" + env_file: + - .env + restart: always diff --git a/target/amavis/conf.d/49-docker-mailserver b/target/amavis/conf.d/49-docker-mailserver new file mode 100644 index 00000000..3d67ae00 --- /dev/null +++ b/target/amavis/conf.d/49-docker-mailserver @@ -0,0 +1,12 @@ +use strict; + +# Override options set in earlier files, use 50-user to override these + +# Bounce spam, the default option for buster is D_PASS to deliver +$final_spam_destiny = D_BOUNCE; + +# Higher log level to get expected messages at startup +$log_level = 2; + +#------------ Do not modify anything below this line ------------- +1; # ensure a defined return diff --git a/target/fail2ban/filter.d/postfix-sasl.conf b/target/fail2ban/filter.d/postfix-sasl.conf new file mode 100644 index 00000000..756b4937 --- /dev/null +++ b/target/fail2ban/filter.d/postfix-sasl.conf @@ -0,0 +1,19 @@ +# Fail2Ban filter for postfix authentication failures + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = postfix(-\w+)?/(?:submission/|smtps/)?smtp[ds] + +failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(:[ A-Za-z0-9+/:]*={0,2})?\s*$ + +ignoreregex = authentication failed: Connection lost to authentication server$ + +[Init] + +journalmatch = _SYSTEMD_UNIT=postfix.service + +ignoreregex = diff --git a/target/filebeat.yml.tmpl b/target/filebeat.yml.tmpl deleted file mode 100644 index d2d84ed4..00000000 --- a/target/filebeat.yml.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -output: - logstash: - enabled: true - hosts: - - $ELK_HOST:$ELK_PORT - -filebeat: - prospectors: - - - paths: - - /var/log/mail/mail.log - document_type: syslog - diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 02c12ac4..ece25c41 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -96,10 +96,6 @@ function register_functions() { _register_setup_function "_setup_default_vars" _register_setup_function "_setup_file_permissions" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_setup_function "_setup_elk_forwarder" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_setup_function "_setup_dovecot" _register_setup_function "_setup_dovecot_dhparam" @@ -212,10 +208,6 @@ function register_functions() { _register_start_daemon "_start_daemons_cron" _register_start_daemon "_start_daemons_rsyslog" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_start_daemon "_start_daemons_filebeat" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_start_daemon "_start_daemons_dovecot" fi @@ -1465,18 +1457,6 @@ function _setup_security_stack() { fi } -function _setup_elk_forwarder() { - notify 'task' 'Setting up Elk forwarder' - - ELK_PORT=${ELK_PORT:="5044"} - ELK_HOST=${ELK_HOST:="elk"} - notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" - cat /etc/filebeat/filebeat.yml.tmpl \ - | sed "s@\$ELK_HOST@$ELK_HOST@g" \ - | sed "s@\$ELK_PORT@$ELK_PORT@g" \ - > /etc/filebeat/filebeat.yml -} - function _setup_logrotate() { notify 'inf' "Setting up logrotate" @@ -1763,11 +1743,6 @@ function _start_daemons_dovecot() { #fi } -function _start_daemons_filebeat() { - notify 'task' 'Starting filebeat' 'n' - supervisorctl start filebeat -} - function _start_daemons_fetchmail() { notify 'task' 'Starting fetchmail' 'n' /usr/local/bin/setup-fetchmail diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 08b246f5..fd010b4d 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -65,14 +65,6 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/dovecot -F -c /etc/dovecot/dovecot.conf -[program:filebeat] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/bin/filebeat -c /etc/filebeat/filebeat.yml - [program:clamav] startsecs=0 autostart=false diff --git a/test/tests.bats b/test/tests.bats index 788bdef5..7d21f509 100644 --- a/test/tests.bats +++ b/test/tests.bats @@ -217,10 +217,8 @@ function count_processed_changes() { } @test "checking smtp: delivers mail to existing account" { - #run docker exec mail /bin/sh -c "grep 'postfix/lmtp' /var/log/mail/mail.log | grep 'status=sent' | grep ' Saved)' | sed 's/.* to=; 6 ; 1 , orig_to=; 1 , orig_to=; 1 ; 2 ;" cat <<'EOF' | assert_output 1 6 @@ -284,7 +282,7 @@ EOF } @test "checking smtp: redirects mail to external aliases" { - run docker exec mail /bin/sh -c "grep -- '-> ' /var/log/mail/mail.log | wc -l" + run docker exec mail /bin/sh -c "grep -- '-> ' /var/log/mail/mail.log* | grep RelayedInbound | wc -l" assert_success assert_output 2 } @@ -785,9 +783,37 @@ EOF } @test "checking system: amavis decoders installed and available" { - run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq | tr '\n' ';'" + run docker exec mail /bin/sh -c "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log*|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq" assert_success - assert_output "7z;Z;arc;arj;bz2;cab;cpio;deb;doc;exe;gz;iso;jar;kmz;lha;lrz;lz4;lzma;lzo;mail;rar;rpm;swf;tar;tnef;xz;zip;zoo;" + # Support for doc and zoo removed in buster + cat <<'EOF' | assert_output +7z +Z +arc +arj +bz2 +cab +cpio +deb +exe +gz +iso +jar +kmz +lha +lrz +lz4 +lzma +lzo +mail +rar +rpm +swf +tar +tnef +xz +zip +EOF } @@ -1307,6 +1333,6 @@ EOF # @test "checking that the container stops cleanly" { - run docker stop -t 60 mail + run docker stop -t 60 mail_override_hostname assert_success }