debian stretch slim (#784)

* Switch to stretch-slim as base image.
 - first step correct the testdata, as newer packages are more strict
about the mail-structure.

* Switch to stretch-slim: correcting the test-environment and the build
 - add missing build-step to make
 - clean the userdb aswell
 - use timeout of netcat, as postgrey would not close the connection
 - there is 2 extra mail-logs -> assert_output 5
 - cosmetic: use "" instead of ''

* Switch to stretch-slim:
new image:
 - smaller size
 - 0 CVEs compared to 11 CVEs in ubuntu 16.04 Image
better backport situation
 - postfix 3.1.6 vs 3.1.0
 - fail2ban 0.9.6 vs 0.9.3
 ...
changes needed because of stretch-slim:
- add missing gnupg and iproute2 package
- remove non-free rar, unrar-free should do
- rsyslog does not add syslog user and has different conf-structure
- pyzor command discover was deprecated and is missing in the new
stretch package

- dovecot does not know SSLv2 anymore. removed because of warnings in
log

- iptables does not know imap3, IMAP working group chose imap2 in favor
of imap3

* Switch to debian stretch slim:
SSLv2 seems to be a not known protocol anymore - good!

* switch to debian stretch slim:
make this test more stable. there might be more than only one mail.log
(mail.info, mail.warn, ...)

* switch to debian stretc slim:
 new openssl 1.1.0 needs stronger ciphers, removed some weekers ones.
Please, look through the new list of cipher! this needs to be done in
another commit for all other SSL/TLS-Endpoints aswell.

* Switch to debian stretch slim:
let our server pre-empt the cipher list.
Did a read through, wwwDOTpostfixDOTorg/FORWARD_SECRECY_READMEDOThtml
and
wwwDOTpostfixDOTorg/TLS_READMEDOThtml

* Switch to debian stretch slim: lets give this openssl-based test a new and independent but identical container.  many other test on the main 'mail' container might interfere here.

* Switch to debian stretch slim: remove unused lines

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM debian:stretch-slim
 MAINTAINER Thomas VIAL
 
 ENV DEBIAN_FRONTEND noninteractive
@@ -41,6 +41,8 @@ RUN apt-get update -q --fix-missing && \
     file \
     gamin \
     gzip \
+    gnupg \
+    iproute2 \
     iptables \
     locales \
     liblz4-tool \
@@ -60,7 +62,6 @@ RUN apt-get update -q --fix-missing && \
     postfix-pcre \
     postfix-policyd-spf-python \
     pyzor \
-    rar \
     razor \
     ripole \
     rpm2cpio \
@@ -133,6 +134,8 @@ COPY target/amavis/conf.d/* /etc/amavis/conf.d/
 RUN sed -i -r 's/#(@|   \\%)bypass/\1bypass/g' /etc/amavis/conf.d/15-content_filter_mode && \
   adduser clamav amavis && \
   adduser amavis clamav && \
+  # no syslog user in debian compared to ubuntu
+  adduser --system syslog && \
   useradd -u 5000 -d /home/docker -s /bin/bash -p $(echo docker | openssl passwd -1 -stdin) docker && \
   (echo "0 4 * * * /usr/local/bin/virus-wiper" ; crontab -l) | crontab -
 
@@ -144,8 +147,7 @@ RUN echo "ignoreregex =" >> /etc/fail2ban/filter.d/postfix-sasl.conf && mkdir /v
 # Enables Pyzor and Razor
 USER amavis
 RUN razor-admin -create && \
-  razor-admin -register && \
+  razor-admin -register
-  pyzor discover
 USER root
 
 # Configure DKIM (opendkim)
@@ -177,8 +179,8 @@ RUN sed -i -r "/^#?compress/c\compress\ncopytruncate" /etc/logrotate.conf && \
   chown -R clamav:root /var/log/mail/clamav.log && \
   touch /var/log/mail/freshclam.log && \
   chown -R clamav:root /var/log/mail/freshclam.log && \
-  sed -i -r 's|/var/log/mail|/var/log/mail/mail|g' /etc/rsyslog.d/50-default.conf && \
+  sed -i -r 's|/var/log/mail|/var/log/mail/mail|g' /etc/rsyslog.conf && \
-  sed -i -r 's|;auth,authpriv.none|;mail.none;mail.error;auth,authpriv.none|g' /etc/rsyslog.d/50-default.conf && \
+  sed -i -r 's|;auth,authpriv.none|;mail.none;mail.error;auth,authpriv.none|g' /etc/rsyslog.conf && \
   sed -i -r 's|LogFile /var/log/clamav/|LogFile /var/log/mail/|g' /etc/clamav/clamd.conf && \
   sed -i -r 's|UpdateLogFile /var/log/clamav/|UpdateLogFile /var/log/mail/|g' /etc/clamav/freshclam.conf && \
   sed -i -r 's|/var/log/clamav|/var/log/mail|g' /etc/logrotate.d/clamav-daemon && \
@@ -205,4 +207,3 @@ EXPOSE 25 587 143 465 993 110 995 4190
 CMD supervisord -c /etc/supervisor/supervisord.conf
 
 ADD target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl
-

Makefile

@@ -1,8 +1,8 @@
 NAME = tvial/docker-mailserver:testing
 
-all: build-no-cache generate-accounts run fixtures tests clean
+all: build-no-cache generate-accounts run generate-accounts-after-run fixtures tests clean
-all-fast: build generate-accounts run fixtures tests clean
+all-fast: build generate-accounts run generate-accounts-after-run fixtures tests clean
-no-build: generate-accounts run fixtures tests clean
+no-build: generate-accounts run generate-accounts-after-run fixtures tests clean
 
 build-no-cache:
 	cd test/docker-openldap/ && docker build -f Dockerfile -t ldap --no-cache .
@@ -36,6 +36,23 @@ run:
 		-e DMS_DEBUG=0 \
 		-h mail.my-domain.com -t $(NAME)
 	sleep 15
+	docker run -d --name mail_privacy \
+		-v "`pwd`/test/config":/tmp/docker-mailserver \
+		-v "`pwd`/test":/tmp/docker-mailserver-test \
+		-e ENABLE_CLAMAV=1 \
+		-e ENABLE_SPAMASSASSIN=1 \
+		-e SA_TAG=-5.0 \
+		-e SA_TAG2=2.0 \
+		-e SA_KILL=3.0 \
+		-e SA_SPAM_SUBJECT="SPAM: " \
+		-e VIRUSMAILS_DELETE_DELAY=7 \
+		-e SASL_PASSWD="external-domain.com username:password" \
+		-e ENABLE_MANAGESIEVE=1 \
+		--cap-add=SYS_PTRACE \
+		-e PERMIT_DOCKER=host \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t $(NAME)
+	sleep 15
 	docker run -d --name mail_pop3 \
 		-v "`pwd`/test/config":/tmp/docker-mailserver \
 		-v "`pwd`/test":/tmp/docker-mailserver-test \
@@ -164,7 +181,7 @@ run:
 generate-accounts-after-run:
 	docker run --rm -e MAIL_USER=added@localhost.localdomain -e MAIL_PASS=mypassword -t $(NAME) /bin/sh -c 'echo "$$MAIL_USER|$$(doveadm pw -s SHA512-CRYPT -u $$MAIL_USER -p $$MAIL_PASS)"' >> test/config/postfix-accounts.cf
 	sleep 10
-    
+
 fixtures:
 	cp config/postfix-accounts.cf config/postfix-accounts.cf.bak
 	# Setup sieve & create filtering folder (INBOX/spam)
@@ -191,7 +208,7 @@ fixtures:
 	docker exec mail_disabled_clamav_spamassassin /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
 	# postfix virtual transport lmtp
 	docker exec mail_lmtp_ip /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
-
+	docker exec mail_privacy /bin/sh -c "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/email-templates/send-privacy-email.txt"
 	docker exec mail_override_hostname /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/existing-user1.txt"
 	# Wait for mails to be analyzed
 	sleep 80
@@ -204,6 +221,7 @@ clean:
 	# Remove running test containers
 	-docker rm -f \
 		mail \
+		mail_privacy \
 		mail_pop3 \
 		mail_smtponly \
 		mail_smtponly_without_config \
@@ -227,4 +245,5 @@ clean:
 		test/config/empty \
 		test/config/without-accounts \
 		test/config/without-virtual \
-		test/config/with-domain
+		test/config/with-domain \
+		test/config/dovecot-lmtp/userdb

target/dovecot/10-ssl.conf

@@ -46,7 +46,7 @@ ssl_key = </etc/dovecot/private/dovecot.pem
 ssl_dh_parameters_length = 2048
 
 # SSL protocols to use
-ssl_protocols = !SSLv2 !SSLv3
+ssl_protocols = !SSLv3
 
 # SSL ciphers to use
 ssl_cipher_list = ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:!DSS

target/fail2ban/jail.conf

@@ -184,7 +184,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
                 %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"]
 
 # Report block via blocklist.de fail2ban reporting service API
-# 
+#
 # See the IMPORTANT note in action.d/blocklist_de.conf for when to
 # use this action. Create a file jail.d/blocklist_de.local containing
 # [Init]
@@ -326,7 +326,7 @@ logpath = /opt/openhab/logs/request.log
 port    = http,https
 logpath = %(nginx_error_log)s
 
-# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` 
+# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module`
 # and define `limit_req` and `limit_req_zone` as described in nginx documentation
 # http://nginx.org/en/docs/http/ngx_http_limit_req_module.html
 # or for example see in 'config/filter.d/nginx-limit-req.conf'
@@ -591,13 +591,13 @@ logpath = /opt/kerio/mailserver/store/logs/security.log
 
 [courier-auth]
 
-port     = smtp,465,submission,imap3,imaps,pop3,pop3s
+port     = smtp,465,submission,imap2,imaps,pop3,pop3s
 logpath  = %(syslog_mail)s
 
 
 [postfix-sasl]
 
-port     = smtp,465,submission,imap3,imaps,pop3,pop3s
+port     = smtp,465,submission,imap2,imaps,pop3,pop3s
 # You might consider monitoring /var/log/mail.warn instead if you are
 # running postfix since it would provide the same log lines at the
 # "warn" level but overall at the smaller filesize.
@@ -607,25 +607,25 @@ enabled  = true
 
 [perdition]
 
-port   = imap3,imaps,pop3,pop3s
+port   = imap2,imaps,pop3,pop3s
 logpath = %(syslog_mail)s
 
 
 [squirrelmail]
 
-port = smtp,465,submission,imap2,imap3,imaps,pop3,pop3s,http,https,socks
+port = smtp,465,submission,imap2,imap2,imaps,pop3,pop3s,http,https,socks
 logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log
 
 
 [cyrus-imap]
 
-port   = imap3,imaps
+port   = imap2,imaps
 logpath = %(syslog_mail)s
 
 
 [uwimap-auth]
 
-port   = imap3,imaps
+port   = imap2,imaps
 logpath = %(syslog_mail)s
 
 

target/postfix/main.cf

@@ -1,6 +1,6 @@
 # See /usr/share/postfix/main.cf.dist for a commented, more complete version
 
-smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+smtpd_banner = $myhostname ESMTP $mail_name (Debian)
 biff = no
 append_dot_mydomain = no
 readme_directory = no
@@ -24,10 +24,12 @@ smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
 #smtp_tls_CAfile=
 smtpd_tls_security_level = may
 smtpd_use_tls=yes
+smtpd_tls_loglevel = 1
 smtp_tls_security_level = may
 smtp_tls_loglevel = 1
 tls_ssl_options = NO_COMPRESSION
-tls_high_cipherlist=ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+tls_high_cipherlist=ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-CCM8:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256
+tls_preempt_cipherlist = yes
 smtpd_tls_protocols=!SSLv2,!SSLv3
 smtp_tls_protocols=!SSLv2,!SSLv3
 smtpd_tls_mandatory_ciphers = high
@@ -79,4 +81,3 @@ policyd-spf_time_limit = 3600
 
 # Remove unwanted headers that reveail our privacy
 smtp_header_checks = pcre:/etc/postfix/maps/sender_header_filter.pcre
-

test/config/dovecot-lmtp/conf.d/10-ssl.conf

@@ -21,7 +21,7 @@ ssl_key = </etc/dovecot/private/dovecot.pem
 # PEM encoded trusted certificate authority. Set this only if you intend to use
 # ssl_verify_client_cert=yes. The file should contain the CA certificate(s)
 # followed by the matching CRL(s). (e.g. ssl_ca = </etc/ssl/certs/ca.pem)
-#ssl_ca = 
+#ssl_ca =
 
 # Require that CRL check succeeds for client certificates.
 #ssl_require_crl = yes
@@ -46,7 +46,7 @@ ssl_key = </etc/dovecot/private/dovecot.pem
 #ssl_dh_parameters_length = 1024
 
 # SSL protocols to use
-ssl_protocols = !SSLv2 !SSLv3
+ssl_protocols = !SSLv3
 
 # SSL ciphers to use
 ssl_cipher_list = ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS

test/email-templates/amavis-spam.txt

@@ -11,4 +11,3 @@ XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
 
 .
 QUIT
-

test/email-templates/amavis-virus.txt

@@ -30,4 +30,3 @@ ACAA/4EAAAAAZWljYXIuY29tUEsFBgAAAAABAAEANwAAAGsAAAAAAA==
 
 .
 QUIT
-

test/email-templates/existing-alias-external.txt

@@ -9,4 +9,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/existing-alias-local.txt

@@ -9,4 +9,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/existing-catchall-local.txt

@@ -9,4 +9,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/existing-user-and-cc-local-alias.txt

@@ -10,4 +10,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/existing-user1.txt

@@ -9,4 +9,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/non-existing-user.txt

@@ -9,4 +9,4 @@ Subject: Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/postgrey.txt

@@ -9,4 +9,4 @@ Subject: Postgrey Test Message
 This is a test mail.
 
 .
-QUIT
+QUIT

test/email-templates/send-privacy-email.txt

@@ -10,5 +10,6 @@ User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:52.0)
  Gecko/20100101 Thunderbird/52.2.1
 Subject: Test ESMTP Auth LOGIN and remove privacy
 testing
+
 .
-quit
+QUIT

test/email-templates/sieve-pipe.txt

@@ -9,4 +9,4 @@ Subject: Sieve pipe test message
 This is a test mail to sieve pipe.
 
 .
-QUIT
+QUIT

test/tests.bats

@@ -166,8 +166,7 @@ load 'test_helper/bats-assert/load'
 }
 
 @test "checking postgrey: there should be a log entry about the whitelisted and passed e-mail user@whitelist.tld in /var/log/mail/mail.log" {
-  run docker exec mail_with_postgrey /bin/sh -c "nc 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt"
+  run docker exec mail_with_postgrey /bin/sh -c "nc -w 8 0.0.0.0 10023 < /tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt"
-  sleep 8
   run docker exec mail_with_postgrey /bin/sh -c "grep -i 'action=pass, reason=client whitelist' /var/log/mail/mail.log | wc -l"
   assert_success
   assert_output 1
@@ -245,7 +244,7 @@ load 'test_helper/bats-assert/load'
 #
 
 @test "checking logs: mail related logs should be located in a subdirectory" {
-  run docker exec mail /bin/sh -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail'|wc -l"
+  run docker exec mail /bin/sh -c "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log'|wc -l"
   assert_success
   assert_output 3
 }
@@ -759,8 +758,8 @@ load 'test_helper/bats-assert/load'
   # Create a container which will send wrong authentications and should banned
   docker run --name fail-auth-mailer -e MAIL_FAIL2BAN_IP=$MAIL_FAIL2BAN_IP -v "$(pwd)/test":/tmp/docker-mailserver-test -d $(docker inspect --format '{{ .Config.Image }}' mail) tail -f /var/log/faillog
 
-  docker exec fail-auth-mailer /bin/sh -c 'nc $MAIL_FAIL2BAN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt'
+  docker exec fail-auth-mailer /bin/sh -c "nc $MAIL_FAIL2BAN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt"
-  docker exec fail-auth-mailer /bin/sh -c 'nc $MAIL_FAIL2BAN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt'
+  docker exec fail-auth-mailer /bin/sh -c "nc $MAIL_FAIL2BAN_IP 25 < /tmp/docker-mailserver-test/auth/smtp-auth-login-wrong.txt"
 
   sleep 5
 
@@ -1199,10 +1198,10 @@ load 'test_helper/bats-assert/load'
 }
 
 @test "checking postfix: remove privacy details of the sender" {
-  run docker exec mail /bin/sh -c "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/email-templates/send-privacy-email.txt | grep 'queued'"
+  run docker exec mail_privacy /bin/sh -c "ls /var/mail/localhost.localdomain/user1/new | wc -l"
   assert_success
-  sleep 10
+  assert_output 1
-  run docker exec mail /bin/sh -c "grep -rE "^User-Agent:" /var/mail/localhost.localdomain/user1/new | wc -l"
+  run docker exec mail_privacy /bin/sh -c "grep -rE "^User-Agent:" /var/mail/localhost.localdomain/user1/new | wc -l"
   assert_success
   assert_output 0
 }