Solve Fetchmail imap idle issue (#10)

* Migrate PR#1730 from tomav/docker-mailserver repo to new
docker-mailserver/docker-mailserver repo
* Resolved review comments
* Moved counter increment to have consistency between fetchmail process
and fetchmail config files
* Added tests for new fetchmail option

Co-authored-by: Georg Lauterbach <44545919+aendeavor@users.noreply.github.com>

ENVIRONMENT.md

@@ -335,6 +335,13 @@ Note: activate this only if you are confident in your bayes database for identif
 
 - **300** => `fetchmail` The number of seconds for the interval
 
+##### FETCHMAIL_PARALLEL
+
+  **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc`
+  **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a seperate fetchmail instance is started  to allow having multiple imap idle configurations defined.
+
+Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances.
+
 #### LDAP
 
 ##### ENABLE_LDAP

target/bin/fetchmailrc_split

@@ -0,0 +1,54 @@
+#! /bin/bash
+
+# Description: This script will split the content of /etc/fetchmailrc into
+#              smaller fetchmailrc files per server [poll] entries. Each
+#              separate fetchmailrc file is stored in /etc/fetchmailrc.d
+#
+#              The mail purpose for this is to work around what is known
+#              as the Fetchmail IMAP idle issue.
+#
+
+FETCHMAILRC="/etc/fetchmailrc"
+FETCHMAILRCD="/etc/fetchmailrc.d"
+DEFAULT_FILE="${FETCHMAILRCD}/defaults"
+
+if [[ ! -r "${FETCHMAILRC}" ]]
+then
+  echo "Error: File ${FETCHMAILRC} not found"
+  exit 1
+fi
+
+if [[ ! -d ${FETCHMAILRCD} ]]
+then
+  if ! mkdir "${FETCHMAILRCD}"
+  then
+    echo "Error: Unable to create folder ${FETCHMAILRCD}"
+    exit 1
+  fi
+fi
+
+COUNTER=0
+SERVER=0
+while read -r LINE
+do
+  if [[ ${LINE} =~ poll ]]
+  then
+    # If we read "poll" then we reached a new server definition
+    # We need to create a new file with fetchmail defaults from
+    # /etc/fetcmailrc
+    COUNTER=$((COUNTER+1))
+    SERVER=1
+    cat "${DEFAULT_FILE}" > "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+    echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+  elif [[ ${SERVER} -eq 0 ]]
+  then
+    # We have not yet found "poll". Let's assume we are still reading
+    # the default settings from /etc/fetchmailrc file
+    echo "${LINE}" >> "${DEFAULT_FILE}"
+  else
+    # Just the server settings that need to be added to the specific rc.d file
+    echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+  fi
+done < "${FETCHMAILRC}"
+
+rm "${DEFAULT_FILE}"

target/scripts/start-mailserver.sh

@@ -18,6 +18,7 @@ ENABLE_SASLAUTHD="${ENABLE_SASLAUTHD:=0}"
 ENABLE_SPAMASSASSIN="${ENABLE_SPAMASSASSIN:=0}"
 ENABLE_SRS="${ENABLE_SRS:=0}"
 FETCHMAIL_POLL="${FETCHMAIL_POLL:=300}"
+FETCHMAIL_PARALLEL="${FETCHMAIL_PARALLEL:=0}"
 LDAP_START_TLS="${LDAP_START_TLS:=no}"
 LOGROTATE_INTERVAL="${LOGROTATE_INTERVAL:=${REPORT_INTERVAL:-daily}}"
 LOGWATCH_INTERVAL="${LOGWATCH_INTERVAL:=none}"
@@ -396,6 +397,7 @@ function _setup_default_vars
     echo "ENABLE_SPAMASSASSIN=${ENABLE_SPAMASSASSIN}"
     echo "ENABLE_SRS=${ENABLE_SRS}"
     echo "FETCHMAIL_POLL=${FETCHMAIL_POLL}"
+    echo "FETCHMAIL_PARALLEL=${FETCHMAIL_PARALLEL}"
     echo "LDAP_START_TLS=${LDAP_START_TLS}"
     echo "LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL}"
     echo "LOGWATCH_INTERVAL=${LOGWATCH_INTERVAL}"
@@ -2068,9 +2070,46 @@ function _start_daemons_dovecot
 
 function _start_daemons_fetchmail
 {
-  _notify 'task' 'Starting fetchmail' 'n'
+  _notify 'task' 'Preparing fetchmail config'
   /usr/local/bin/setup-fetchmail
+  if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]
+  then
+    mkdir /etc/fetchmailrc.d/
+    /usr/local/bin/fetchmailrc_split
+
+    COUNTER=0
+    for RC in /etc/fetchmailrc.d/fetchmail-*.rc
+    do
+      COUNTER=$((COUNTER+1))
+      cat <<EOF > "/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf"
+[program:fetchmail-${COUNTER}]
+startsecs=0
+autostart=false
+autorestart=true
+stdout_logfile=/var/log/supervisor/%(program_name)s.log
+stderr_logfile=/var/log/supervisor/%(program_name)s.log
+user=fetchmail
+command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid
+EOF
+      chmod 700 "${RC}"
+      chown fetchmail:root "${RC}"
+    done
+
+    supervisorctl reread
+    supervisorctl update
+
+    COUNTER=0
+    for _ in /etc/fetchmailrc.d/fetchmail-*.rc
+    do
+      COUNTER=$((COUNTER+1))
+      _notify 'task' "Starting fetchmail instance ${COUNTER}" 'n'
+      supervisorctl start "fetchmail-${COUNTER}"
+    done
+
+  else
+    _notify 'task' 'Starting fetchmail' 'n'
     supervisorctl start fetchmail
+  fi
 }
 
 function _start_daemons_clamav

test/config/fetchmail.cf

@@ -3,3 +3,9 @@ poll pop3.example.com. with proto POP3
 	password 'secret'
 	is 'user2@domain.tld'
 	here options keep ssl
+	
+poll pop3-2.example.com. with proto POP3
+	user 'username' there with
+	password 'secret'
+	is 'user3@domain.tld'
+	here options keep ssl

test/mail_fetchmail_parallel.bats

@@ -0,0 +1,97 @@
+load 'test_helper/common'
+
+function setup() {
+    run_setup_file_if_necessary
+}
+
+function teardown() {
+    run_teardown_file_if_necessary
+}
+
+function setup_file() {
+    local PRIVATE_CONFIG
+    PRIVATE_CONFIG="$(duplicate_config_for_container .)"
+    docker run -d --name mail_fetchmail_parallel \
+		-v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \
+		-v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \
+		-e ENABLE_FETCHMAIL=1 \
+		-e FETCHMAIL_PARALLEL=1 \
+		--cap-add=NET_ADMIN \
+		-e DMS_DEBUG=0 \
+		-h mail.my-domain.com -t "${NAME}"
+    wait_for_finished_setup_in_container mail_fetchmail_parallel
+}
+
+function teardown_file() {
+    docker rm -f mail_fetchmail_parallel
+}
+
+@test "first" {
+  skip 'this test must come first to reliably identify when to run setup_file'
+}
+
+#
+# processes
+#
+
+@test "checking process: fetchmail 1 (fetchmail server enabled)" {
+  run docker exec mail_fetchmail_parallel /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-1.rc'"
+  assert_success
+}
+
+@test "checking process: fetchmail 2 (fetchmail server enabled)" {
+  run docker exec mail_fetchmail_parallel /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-2.rc'"
+  assert_success
+}
+
+#
+# fetchmail
+#
+
+@test "checking fetchmail: gerneral options in fetchmail-1.rc are loaded" {
+  run docker exec mail_fetchmail_parallel grep 'set syslog' /etc/fetchmailrc.d/fetchmail-1.rc
+  assert_success
+}
+
+@test "checking fetchmail: gerneral options in fetchmail-2.rc are loaded" {
+  run docker exec mail_fetchmail_parallel grep 'set syslog' /etc/fetchmailrc.d/fetchmail-2.rc
+  assert_success
+}
+
+@test "checking fetchmail: fetchmail-1.rc is loaded with pop3.example.com" {
+  run docker exec mail_fetchmail_parallel grep 'pop3.example.com' /etc/fetchmailrc.d/fetchmail-1.rc
+  assert_success
+}
+
+@test "checking fetchmail: fetchmail-1.rc is loaded without pop3-2.example.com" {
+  run docker exec mail_fetchmail_parallel grep 'pop3-2.example.com' /etc/fetchmailrc.d/fetchmail-1.rc
+  assert_failure
+}
+
+@test "checking fetchmail: fetchmail-2.rc is loaded without pop3.example.com" {
+  run docker exec mail_fetchmail_parallel grep 'pop3.example.com' /etc/fetchmailrc.d/fetchmail-2.rc
+  assert_failure
+}
+
+@test "checking fetchmail: fetchmail-2.rc is loaded with pop3-2.example.com" {
+  run docker exec mail_fetchmail_parallel grep 'pop3-2.example.com' /etc/fetchmailrc.d/fetchmail-2.rc
+  assert_success
+}
+
+#
+# supervisor
+#
+
+@test "checking restart of process: fetchmail-1" {
+  run docker exec mail_fetchmail_parallel /bin/bash -c "pkill fetchmail && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-1.rc'"
+  assert_success
+}
+
+@test "checking restart of process: fetchmail-2" {
+  run docker exec mail_fetchmail_parallel /bin/bash -c "pkill fetchmail && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-2.rc'"
+  assert_success
+}
+
+@test "last" {
+  skip 'this test is only there to reliably mark the end for the teardown_file'
+}