Implementation of multi-domain relay hosts (#922, #926)

* Add new configuration for multi-domain relay hosts (#922)
 * Creates new environment variables (replacing existing AWS_SES variables)
 * Optionally allows more advanced setups using config files
* Update relay hosts during change detection (#922)
* Add helper scripts for adding relay hosts and per-domain auth
* Allow the possibility to deliver some mail directly
* adding a domain with no destination will exclude it from the
  relayhost_map and so Postfix will attempt to deliver the mail directly
* tests for setup.sh script
* tests for relay host configuration
* these tests cover the code in `start-mailserver.sh` dealing with both
  the env vars and the configuration files

.env.dist

@@ -273,3 +273,26 @@ SRS_EXCLUDE_DOMAINS=
 # signing and the remaining will be used for verification. this is how you
 # rotate and expire keys
 SRS_SECRET=
+
+# -----------------------------------------------------------------------------------------------------------------------------
+# ---------------- Multi-domain relay section ---------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------------------------------------------------------
+
+# Setup relaying for multiple domains based on the domain name of the sender
+# optionally uses usernames and passwords in postfix-sasl-password.cf and relay host mappings in postfix-relaymap.cf
+#
+# empty => don't configure relay host
+# default host to relay mail through
+RELAY_HOST=
+
+# empty => 25
+# default port to relay mail
+RELAY_PORT=25
+
+# empty => no default
+# default relay username (if no specific entry exists in postfix-sasl-password.cf)
+RELAY_USER=
+
+# empty => no default
+# password for default relay user
+RELAY_PASSWORD=

Makefile

@@ -207,7 +207,18 @@ run:
 		-e SA_SPAM_SUBJECT="undef" \
 		-h mail.my-domain.com -t $(NAME)
 	sleep 15
-
+	docker run -d --name mail_with_relays \
+		-v "`pwd`/test/config/relay-hosts":/tmp/docker-mailserver \
+		-v "`pwd`/test":/tmp/docker-mailserver-test \
+		-e RELAY_HOST=default.relay.com \
+		-e RELAY_PORT=2525 \
+		-e RELAY_USER=smtp_user \
+		-e RELAY_PASSWORD=smtp_password \
+		--cap-add=SYS_PTRACE \
+		-e PERMIT_DOCKER=host \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t $(NAME)
+	sleep 15
 
 generate-accounts-after-run:
 	docker run --rm -e MAIL_USER=added@localhost.localdomain -e MAIL_PASS=mypassword -t $(NAME) /bin/sh -c 'echo "$$MAIL_USER|$$(doveadm pw -s SHA512-CRYPT -u $$MAIL_USER -p $$MAIL_PASS)"' >> test/config/postfix-accounts.cf
@@ -274,7 +285,7 @@ clean:
 		mv config.bak config ;\
 	fi
 	@if [ -d testconfig.bak ]; then\
-		rm -rf test/config ;\
+		sudo rm -rf test/config ;\
 		mv testconfig.bak test/config ;\
 	fi
 	-sudo rm -rf test/onedir

README.md

@@ -538,3 +538,25 @@ Note: This postgrey setting needs `ENABLE_POSTGREY=1`
   - you may specify multiple keys, comma separated. the first one is used for signing and the remaining will be used for verification. this is how you rotate and expire keys
   - if you have a cluster/swarm make sure the same keys are on all nodes
   - example command to generate a key: `dd if=/dev/urandom bs=24 count=1 2>/dev/null | base64`
+
+## Multi-domain Relay Hosts
+
+#### RELAY_HOST
+
+  - **empty** => don't configure relay host
+  - default host to relay mail through
+
+#### RELAY_PORT
+
+  - **empty** => 25
+  - default port to relay mail through
+
+#### RELAY_USER
+
+  - **empty** => no default
+  - default relay username (if no specific entry exists in postfix-sasl-password.cf)
+
+#### RELAY_PASSWORD
+
+  - **empty** => no default
+  - password for default relay user

setup.sh

@@ -61,6 +61,12 @@ SUBCOMMANDS:
     $0 config dkim <keysize> (default: 2048)
     $0 config ssl
 
+  relay:
+
+    $0 relay add-domain <domain> <host> [<port>]
+    $0 relay add-auth <domain> <username> [<password>]
+    $0 relay exclude-domain <domain>
+
   debug:
 
     $0 debug fetchmail
@@ -197,6 +203,27 @@ case $1 in
     esac
     ;;
 
+  relay)
+    shift
+    case $1 in
+      add-domain)
+        shift
+        _docker_image addrelayhost $@
+        ;;
+      add-auth)
+        shift
+        _docker_image addsaslpassword $@
+        ;;
+      exclude-domain)
+        shift
+        _docker_image excluderelaydomain $@
+        ;;
+      *)
+        _usage
+        ;;
+    esac
+    ;;
+
   debug)
     shift
     case $1 in

target/bin/addrelayhost

@@ -0,0 +1,33 @@
+#! /bin/bash
+
+DATABASE=${DATABASE:-/tmp/docker-mailserver/postfix-relaymap.cf}
+
+DOMAIN="$1"
+HOST="$2"
+PORT="$3"
+
+usage() {
+	echo "Usage: addrelayhost <domain> <host> [<port>]"
+}
+
+errex() {
+	echo "$@" 1>&2
+	exit 1
+}
+
+escape() {
+	echo "${1//./\\.}"
+}
+
+[ -z "$DOMAIN" ] && { usage; errex "no domain specified"; }
+[ -z "$HOST" ] && { usage; errex "no relay host specified"; }
+
+if [ -z "$PORT" ]; then
+	PORT=25
+fi
+
+if grep -qi "^@$DOMAIN" $DATABASE 2>/dev/null; then
+	sed -i "s ^@"$DOMAIN".* "@$DOMAIN"\t\t["$HOST"]:"$PORT" " $DATABASE
+else
+	echo -e "@$DOMAIN\t\t[$HOST]:$PORT" >> $DATABASE
+fi

target/bin/addsaslpassword

@@ -0,0 +1,35 @@
+#! /bin/bash
+
+DATABASE=${DATABASE:-/tmp/docker-mailserver/postfix-sasl-password.cf}
+
+DOMAIN="$1"
+USER="$2"
+PASSWD="$3"
+
+usage() {
+	echo "Usage: addsaslpassword <domain> <username> <password>"
+}
+
+errex() {
+	echo "$@" 1>&2
+	exit 1
+}
+
+escape() {
+	echo "${1//./\\.}"
+}
+
+[ -z "$DOMAIN" ] && { usage; errex "no domain specified"; }
+[ -z "$USER" ] && { usage; errex "no username specified"; }
+
+if [ -z "$PASSWD" ]; then
+	read -s -p "Enter Password: " PASSWD
+	echo
+	[ -z "$PASSWD" ] && errex "Password must not be empty"
+fi
+
+if grep -qi "^@$DOMAIN" $DATABASE 2>/dev/null; then
+	sed -i "s ^@"$DOMAIN".* "@$DOMAIN"\t\t"$USER":"$PASSWD" " $DATABASE
+else
+	echo -e "@$DOMAIN\t\t$USER:$PASSWD" >> $DATABASE
+fi

target/bin/excluderelaydomain

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+DATABASE=${DATABASE:-/tmp/docker-mailserver/postfix-relaymap.cf}
+
+DOMAIN="$1"
+
+usage() {
+	echo "Usage: excluderelayhost <domain>"
+}
+
+errex() {
+	echo "$@" 1>&2
+	exit 1
+}
+
+[ -z "$DOMAIN" ] && { usage; errex "no domain specified"; }
+
+if grep -qi "^@$DOMAIN" $DATABASE 2>/dev/null; then
+	sed -i "s/^@"$DOMAIN".*/@"$DOMAIN"/" $DATABASE
+else
+	echo -e "@$DOMAIN" >> $DATABASE
+fi

target/check-for-changes.sh

@@ -51,6 +51,37 @@ if ! [ $resu_acc = "OK" ] || ! [ $resu_vir = "OK" ]; then
 		chmod 640 /etc/dovecot/userdb
 		sed -i -e '/\!include auth-ldap\.conf\.ext/s/^/#/' /etc/dovecot/conf.d/10-auth.conf
 		sed -i -e '/\!include auth-passwdfile\.inc/s/^#//' /etc/dovecot/conf.d/10-auth.conf
+
+		# rebuild relay host
+		if [ ! -z "$RELAY_HOST" ]; then
+			# keep old config
+			echo -n > /etc/postfix/sasl_passwd
+			echo -n > /etc/postfix/relayhost_map
+			if [ ! -z "$SASL_PASSWD" ]; then
+				echo "$SASL_PASSWD" >> /etc/postfix/sasl_passwd
+			fi
+			# add domain-specific auth from config file
+			if [ -f /tmp/docker-mailserver/postfix-sasl-password.cf ]; then
+				while read line; do
+					if ! echo "$line" | grep -q -e "\s*#"; then
+						echo "$line" >> /etc/postfix/sasl_passwd
+					fi
+				done < /tmp/docker-mailserver/postfix-sasl-password.cf
+			fi
+			# add default relay
+			if [ ! -z "$RELAY_USER" ] && [ ! -z "$RELAY_PASSWORD" ]; then
+				echo "[$RELAY_HOST]:$RELAY_PORT		$RELAY_USER:$RELAY_PASSWORD" >> /etc/postfix/sasl_passwd
+			fi
+			# add relay maps from file
+			if [ -f /tmp/docker-mailserver/postfix-relaymap.cf ]; then
+				while read line; do
+					if ! echo "$line" | grep -q -e "\s*#"; then
+						echo "$line" >> /etc/postfix/relayhost_map
+					fi
+				done < /tmp/docker-mailserver/postfix-relaymap.cf
+			fi
+		fi
+
 		# Creating users
 		# 'pass' is encrypted
 		# comments and empty lines are ignored
@@ -78,8 +109,22 @@ if ! [ $resu_acc = "OK" ] || ! [ $resu_vir = "OK" ]; then
 			# Copy user provided sieve file, if present
 			test -e /tmp/docker-mailserver/${login}.dovecot.sieve && cp /tmp/docker-mailserver/${login}.dovecot.sieve /var/mail/${domain}/${user}/.dovecot.sieve
 			echo ${domain} >> /tmp/vhost.tmp
+			# add domains to relayhost_map
+			if [ ! -z "$RELAY_HOST" ]; then
+				if ! grep -q -e "^@${domain}\s" /etc/postfix/relayhost_map; then
+					echo "@${domain}		[$RELAY_HOST]:$RELAY_PORT" >> /etc/postfix/relayhost_map
+				fi
+			fi
 		done
 	fi
+	if [ -f /etc/postfix/sasl_passwd ]; then
+		chown root:root /etc/postfix/sasl_passwd
+		chmod 0600 /etc/postfix/sasl_passwd
+	fi
+	if [ -f /etc/postfix/relayhost_map ]; then
+		chown root:root /etc/postfix/relayhost_map
+		chmod 0600 /etc/postfix/relayhost_map
+	fi
 	if [ -f postfix-virtual.cf ]; then
     # regen postfix aliases
     echo -n > /etc/postfix/virtual

target/start-mailserver.sh

@@ -135,7 +135,11 @@ function register_functions() {
   _register_setup_function "_setup_postfix_access_control"
 
 	if [ ! -z "$AWS_SES_HOST" -a ! -z "$AWS_SES_USERPASS" ]; then
-		_register_setup_function "_setup_postfix_relay_amazon_ses"
+		_register_setup_function "_setup_postfix_relay_hosts"
+	fi
+
+	if [ ! -z "$RELAY_HOST" ]; then
+		_register_setup_function "_setup_postfix_relay_hosts"
 	fi
 
 	if [ "$ENABLE_POSTFIX_VIRTUAL_TRANSPORT" = 1  ]; then
@@ -1001,22 +1005,102 @@ function _setup_postfix_sasl_password() {
 	fi
 }
 
-function _setup_postfix_relay_amazon_ses() {
+function _setup_postfix_relay_hosts() {
-	notify 'task' 'Setting up Postfix Relay Amazon SES'
+	notify 'task' 'Setting up Postfix Relay Hosts'
-	if [ -z "$AWS_SES_PORT" ];then
+	# copy old AWS_SES variables to new variables
-		AWS_SES_PORT=25
+	if [ -z "$RELAY_HOST" ]; then
+		if [ ! -z "$AWS_SES_HOST" ]; then
+			notify 'inf' "Using deprecated AWS_SES environment variables"
+			RELAY_HOST=$AWS_SES_HOST
 		fi
-	notify 'inf' "Setting up outgoing email via AWS SES host $AWS_SES_HOST:$AWS_SES_PORT"
+	fi
-	echo "[$AWS_SES_HOST]:$AWS_SES_PORT $AWS_SES_USERPASS" >> /etc/postfix/sasl_passwd
+	if [ -z "$RELAY_PORT" ]; then
+		if [ -z "$AWS_SES_PORT" ]; then
+			RELAY_PORT=25
+		else
+			RELAY_PORT=$AWS_SES_PORT
+		fi
+	fi
+	if [ -z "$RELAY_USER" ]; then
+		if [ ! -z "$AWS_SES_USERPASS" ]; then
+			# NB this will fail if the password contains a colon!
+			RELAY_USER=$(echo "$AWS_SES_USERPASS" | cut -f 1 -d ":")
+			RELAY_PASSWORD=$(echo "$AWS_SES_USERPASS" | cut -f 2 -d ":")
+		fi
+	fi
+	notify 'inf' "Setting up outgoing email relaying via $RELAY_HOST:$RELAY_PORT"
+
+	# setup /etc/postfix/sasl_passwd
+	# --
+	# @domain1.com        postmaster@domain1.com:your-password-1
+	# @domain2.com        postmaster@domain2.com:your-password-2
+	# @domain3.com        postmaster@domain3.com:your-password-3
+	#
+	# [smtp.mailgun.org]:587  postmaster@domain2.com:your-password-2
+
+	if [ -f /tmp/docker-mailserver/postfix-sasl-password.cf ]; then
+		notify 'inf' "Adding relay authentication from postfix-sasl-password.cf"
+		while read line; do
+			if ! echo "$line" | grep -q -e "\s*#"; then
+				echo "$line" >> /etc/postfix/sasl_passwd
+			fi
+		done < /tmp/docker-mailserver/postfix-sasl-password.cf
+	fi
+
+	# add default relay
+	if [ ! -z "$RELAY_USER" ] && [ ! -z "$RELAY_PASSWORD" ]; then
+		echo "[$RELAY_HOST]:$RELAY_PORT		$RELAY_USER:$RELAY_PASSWORD" >> /etc/postfix/sasl_passwd
+	else
+		if [ ! -f /tmp/docker-mailserver/postfix-sasl-password.cf ]; then
+			notify 'warn' "No relay auth file found and no default set"
+		fi
+	fi
+
+	chown root:root /etc/postfix/sasl_passwd
+	chmod 0600 /etc/postfix/sasl_passwd
+	# end /etc/postfix/sasl_passwd
+
+	# setup /etc/postfix/relayhost_map
+	# --
+	# @domain1.com        [smtp.mailgun.org]:587
+	# @domain2.com        [smtp.mailgun.org]:587
+	# @domain3.com        [smtp.mailgun.org]:587
+
+	echo -n > /etc/postfix/relayhost_map
+
+	if [ -f /tmp/docker-mailserver/postfix-relaymap.cf ]; then
+		notify 'inf' "Adding relay mappings from postfix-relaymap.cf"
+		while read line; do
+			if ! echo "$line" | grep -q -e "\s*#"; then
+				echo "$line" >> /etc/postfix/relayhost_map
+			fi
+		done < /tmp/docker-mailserver/postfix-relaymap.cf
+	fi
+	grep -v "^\s*$\|^\s*\#" /tmp/docker-mailserver/postfix-accounts.cf | while IFS=$'|' read login pass
+	do
+		domain=$(echo ${login} | cut -d @ -f2)
+		if ! grep -q -e "^@${domain}\b" /etc/postfix/relayhost_map; then
+			notify 'inf' "Adding relay mapping for ${domain}"
+			echo "@${domain}		[$RELAY_HOST]:$RELAY_PORT" >> /etc/postfix/relayhost_map
+		fi
+	done
+	# remove lines with no destination
+	sed -i '/^@\S*\s*$/d' /etc/postfix/relayhost_map
+
+	chown root:root /etc/postfix/relayhost_map
+	chmod 0600 /etc/postfix/relayhost_map
+	# end /etc/postfix/relayhost_map
+
 	postconf -e \
-		"relayhost = [$AWS_SES_HOST]:$AWS_SES_PORT" \
 		"smtp_sasl_auth_enable = yes" \
 		"smtp_sasl_security_options = noanonymous" \
 		"smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd" \
 		"smtp_use_tls = yes" \
 		"smtp_tls_security_level = encrypt" \
 		"smtp_tls_note_starttls_offer = yes" \
-		"smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt"
+		"smtp_tls_CAfile = /etc/ssl/certs/ca-certificates.crt" \
+		"sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map" \
+		"smtp_sender_dependent_authentication = yes"
 }
 
 function _setup_postfix_dhparam() {

test/config/relay-hosts/postfix-accounts.cf

@@ -0,0 +1,3 @@
+user1@domainone.tld|{SHA512-CRYPT}$6$UMGnThsSm0IFgzEw$BynVshxudpGQHDRQaF4b7wb57A7NazGZcBUakYYLflp7J4E3UHK2qo/C1qXMCkRlYFlTd.SuwCsCKb7zBaUkb/
+user2@domaintwo.tld|{SHA512-CRYPT}$6$pDkaJmqaNJB0GNtl$692CLTUW1Z2C3kV1c9geJ7jWRrxqtT3eE7d/EEufEU2hI0br.SbLTItIfKPO9vhqpgWYB/bmUM7VRs7UZbN1w.
+user3@domainthree.tld|{SHA512-CRYPT}$6$y370fIl7h8PUWXkp$aC2Pp7xw2Gc9CSQW0cYORFfAOjzBJB8iu/GDFe9D4PVnE0aupFGdupm9db.7SU8Ur9T4eZyJ75.Be747XdDZ.0

test/config/relay-hosts/postfix-relaymap.cf

@@ -0,0 +1,2 @@
+@domaintwo.tld		[other.relay.com]:587
+@domainthree.tld

test/config/relay-hosts/postfix-sasl-password.cf

@@ -0,0 +1 @@
+@domaintwo.tld		smtp_user_2:smtp_password_2

test/tests.bats

@@ -1310,6 +1310,46 @@ load 'test_helper/bats-assert/load'
   assert_output --partial "You need to specify an IP address. Run"
 }
 
+@test "checking setup.sh: setup.sh relay add-domain" {
+  echo -n > ./config/postfix-relaymap.cf
+  ./setup.sh -c mail relay add-domain example1.org smtp.relay1.com 2525
+  ./setup.sh -c mail relay add-domain example2.org smtp.relay2.com
+  ./setup.sh -c mail relay add-domain example3.org smtp.relay3.com 2525
+  ./setup.sh -c mail relay add-domain example3.org smtp.relay.com 587
+
+  # check adding
+  run /bin/sh -c 'cat ./config/postfix-relaymap.cf | grep -e "^@example1.org\s\+\[smtp.relay1.com\]:2525" | wc -l | grep 1'
+  assert_success
+  # test default port
+  run /bin/sh -c 'cat ./config/postfix-relaymap.cf | grep -e "^@example2.org\s\+\[smtp.relay2.com\]:25" | wc -l | grep 1'
+  assert_success
+  # test modifying
+  run /bin/sh -c 'cat ./config/postfix-relaymap.cf | grep -e "^@example3.org\s\+\[smtp.relay.com\]:587" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking setup.sh: setup.sh relay add-auth" {
+  echo -n > ./config/postfix-sasl-password.cf
+  ./setup.sh -c mail relay add-auth example.org smtp_user smtp_pass
+  ./setup.sh -c mail relay add-auth example2.org smtp_user2 smtp_pass2
+  ./setup.sh -c mail relay add-auth example2.org smtp_user2 smtp_pass_new
+
+  # test adding
+  run /bin/sh -c 'cat ./config/postfix-sasl-password.cf | grep -e "^@example.org\s\+smtp_user:smtp_pass" | wc -l | grep 1'
+  assert_success
+  # test updating
+  run /bin/sh -c 'cat ./config/postfix-sasl-password.cf | grep -e "^@example2.org\s\+smtp_user2:smtp_pass_new" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking setup.sh: setup.sh relay exclude-domain" {
+  echo -n > ./config/postfix-relaymap.cf
+  ./setup.sh -c mail relay exclude-domain example.org
+
+  run /bin/sh -c 'cat ./config/postfix-relaymap.cf | grep -e "^@example.org\s*$" | wc -l | grep 1'
+  assert_success
+}
+
 #
 # LDAP
 #
@@ -1617,3 +1657,32 @@ load 'test_helper/bats-assert/load'
   run docker exec mail_with_ldap /bin/bash -c "pkill saslauthd && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/sbin/saslauthd'"
   assert_success
 }
+
+#
+# relay hosts
+#
+
+@test "checking relay hosts: default mapping is added from env vars" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainone.tld\s\+\[default.relay.com\]:2525" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: custom mapping is added from file" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domaintwo.tld\s\+\[other.relay.com\]:587" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: ignored domain is not added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/relayhost_map | grep -e "^@domainthree.tld\s\+\[any.relay.com\]:25" | wc -l | grep 0'
+  assert_success
+}
+
+@test "checking relay hosts: auth entry is added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^@domaintwo.tld\s\+smtp_user_2:smtp_password_2" | wc -l | grep 1'
+  assert_success
+}
+
+@test "checking relay hosts: default auth entry is added" {
+  run docker exec mail_with_relays /bin/sh -c 'cat /etc/postfix/sasl_passwd | grep -e "^\[default.relay.com\]:2525\s\+smtp_user:smtp_password" | wc -l | grep 1'
+  assert_success
+}