scripts: refactored daemon-stack.sh (#2496)

* refactored `daemon-stack.sh`

A new method was introduced to uniformaly start daemons and log output
accordingly. The methods for daemon start were renamed (plural ->
singular), therefore the adjustments in `start-mailserver.sh`.

* cleaned Fetchmail setup from `daemon-stack.sh`

Not sure why, but the Fetchmail setup was somehow happening in
`daemon-stack.sh` - this is not supposed to be the case. I relocated the
setup into `setup-stack.sh`, where it belong.

* delete old, unnecessary script in `target/bin/`

These are unused leftovers from the last commit, that relocated the
setup of Fetchmail into `setup.stack.sh`.

* corrected changedetector function name

* Apply suggestions from code review

* adjusted `debug-fetchmail` script

It is absolutely fine to source `setup-stack.sh` because sourcing the
script does not execute a single function (by desing of the script).
This way, we retain functionality.

* praise be ShellCheck

* added `log.sh` to `debug-fetchmail` as a dependency

* final cleanup

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>
Co-authored-by: Casper <casperklein@users.noreply.github.com>

target/bin/debug-fetchmail

@@ -1,6 +1,11 @@
 #! /bin/bash
 
-/usr/local/bin/setup-fetchmail
+# shellcheck source=../scripts/helpers/log.sh
+source /usr/local/bin/helpers/log.sh
+# shellcheck source=../scripts/startup/setup-stack.sh
+source /usr/local/bin/setup-stack.sh
+
+_setup_fetchmail
 
 su -s /bin/sh -c "/usr/bin/fetchmail \
   --verbose \

target/bin/fetchmailrc_split

@@ -1,55 +0,0 @@
-#! /bin/bash
-
-# Description: This script will split the content of /etc/fetchmailrc into
-#              smaller fetchmailrc files per server [poll] entries. Each
-#              separate fetchmailrc file is stored in /etc/fetchmailrc.d
-#
-#              The mail purpose for this is to work around what is known
-#              as the Fetchmail IMAP idle issue.
-#
-
-FETCHMAILRC="/etc/fetchmailrc"
-FETCHMAILRCD="/etc/fetchmailrc.d"
-DEFAULT_FILE="${FETCHMAILRCD}/defaults"
-
-if [[ ! -r "${FETCHMAILRC}" ]]
-then
-  _log 'error' "File '${FETCHMAILRC}' not found"
-  exit 1
-fi
-
-if [[ ! -d ${FETCHMAILRCD} ]]
-then
-  if ! mkdir "${FETCHMAILRCD}"
-  then
-    _log 'error' "Unable to create folder '${FETCHMAILRCD}'"
-    exit 1
-  fi
-fi
-
-COUNTER=0
-SERVER=0
-while read -r LINE
-do
-  if [[ ${LINE} =~ poll ]]
-  then
-    # If we read "poll" then we reached a new server definition
-    # We need to create a new file with fetchmail defaults from
-    # /etc/fetcmailrc
-    COUNTER=$((COUNTER+1))
-    SERVER=1
-    cat "${DEFAULT_FILE}" > "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
-    echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
-  elif [[ ${SERVER} -eq 0 ]]
-  then
-    # We have not yet found "poll". Let's assume we are still reading
-    # the default settings from /etc/fetchmailrc file
-    echo "${LINE}" >> "${DEFAULT_FILE}"
-  else
-    # Just the server settings that need to be added to the specific rc.d file
-    echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
-  fi
-# delete commented lines before parsing
-done < <(sed '/^[[:space:]]*#/d' "${FETCHMAILRC}")
-
-rm "${DEFAULT_FILE}"

target/bin/setup-fetchmail

@@ -1,14 +0,0 @@
-#! /bin/bash
-
-CONF=/tmp/docker-mailserver/fetchmail.cf
-RC=/etc/fetchmailrc
-
-if [[ -f ${CONF} ]]
-then
-  cat /etc/fetchmailrc_general "${CONF}" >"${RC}"
-else
-  cat /etc/fetchmailrc_general >"${RC}"
-fi
-
-chmod 700 "${RC}"
-chown fetchmail:root "${RC}"

target/scripts/start-mailserver.sh

@@ -170,10 +170,16 @@ function register_functions
   # needs to come after _setup_postfix_aliases
   [[ ${SPOOF_PROTECTION} -eq 1 ]] && _register_setup_function '_setup_spoof_protection'
 
+  if [[ ${ENABLE_FETCHMAIL} -eq 1 ]]
+  then
+    _register_setup_function '_setup_fetchmail'
+    [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] && _register_setup_function '_setup_fetchmail_parallel'
+  fi
+
   if [[ ${ENABLE_SRS} -eq 1  ]]
   then
     _register_setup_function '_setup_SRS'
-    _register_start_daemon '_start_daemons_postsrsd'
+    _register_start_daemon '_start_daemon_postsrsd'
   fi
 
   _register_setup_function '_setup_postfix_access_control'
@@ -204,28 +210,28 @@ function register_functions
 
   # ? >> Daemons
 
-  _register_start_daemon '_start_daemons_cron'
+  _register_start_daemon '_start_daemon_cron'
-  _register_start_daemon '_start_daemons_rsyslog'
+  _register_start_daemon '_start_daemon_rsyslog'
 
-  [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemons_dovecot'
+  [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot'
-  [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemons_update_check'
+  [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check'
 
   # needs to be started before SASLauthd
-  _register_start_daemon '_start_daemons_opendkim'
+  _register_start_daemon '_start_daemon_opendkim'
-  _register_start_daemon '_start_daemons_opendmarc'
+  _register_start_daemon '_start_daemon_opendmarc'
 
   # needs to be started before postfix
-  [[ ${ENABLE_POSTGREY} -eq 1 ]] &&	_register_start_daemon '_start_daemons_postgrey'
+  [[ ${ENABLE_POSTGREY} -eq 1 ]] &&	_register_start_daemon '_start_daemon_postgrey'
 
-  _register_start_daemon '_start_daemons_postfix'
+  _register_start_daemon '_start_daemon_postfix'
 
   # needs to be started after postfix
-  [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && _register_start_daemon '_start_daemons_saslauthd'
+  [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && _register_start_daemon '_start_daemon_saslauthd'
-  [[ ${ENABLE_FAIL2BAN} -eq 1 ]] &&	_register_start_daemon '_start_daemons_fail2ban'
+  [[ ${ENABLE_FAIL2BAN} -eq 1 ]] &&	_register_start_daemon '_start_daemon_fail2ban'
-  [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && _register_start_daemon '_start_daemons_fetchmail'
+  [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && _register_start_daemon '_start_daemon_fetchmail'
-  [[ ${ENABLE_CLAMAV} -eq 1 ]] &&	_register_start_daemon '_start_daemons_clamav'
+  [[ ${ENABLE_CLAMAV} -eq 1 ]] &&	_register_start_daemon '_start_daemon_clamav'
-  [[ ${ENABLE_LDAP} -eq 0 ]] && _register_start_daemon '_start_changedetector'
+  [[ ${ENABLE_LDAP} -eq 0 ]] && _register_start_daemon '_start_daemon_changedetector'
-  [[ ${ENABLE_AMAVIS} -eq 1 ]] && _register_start_daemon '_start_daemons_amavis'
+  [[ ${ENABLE_AMAVIS} -eq 1 ]] && _register_start_daemon '_start_daemon_amavis'
 }
 
 function _register_start_daemon
@@ -275,7 +281,7 @@ setup
 [[ ${LOG_LEVEL} =~ (debug|trace) ]] && print-environment
 fix
 start_misc
-start_daemons
+_start_daemons
 
 # marker to check, if container was restarted
 date >/CONTAINER_START

target/scripts/startup/daemons-stack.sh

@@ -1,160 +1,86 @@
 #! /bin/bash
 
-function start_daemons
+function _start_daemons
 {
-  _log 'info' 'Starting daemons & mail server'
+  _log 'info' 'Starting daemons'
-  for FUNC in "${DAEMONS_START[@]}"
+
+  for FUNCTION in "${DAEMONS_START[@]}"
   do
-    ${FUNC}
+    ${FUNCTION}
   done
 }
 
-function _start_daemons_cron
+function _default_start_daemon
 {
-  _log 'debug' 'Starting cron'
+  _log 'debug' "Starting ${1:?}"
-  supervisorctl start cron || dms_panic__fail_init 'cron'
+
+  local RESULT
+  RESULT="$(supervisorctl start "${1}" 2>&1)"
+
+  # shellcheck disable=SC2181
+  if [[ ${?} -ne 0 ]]
+  then
+    echo "${RESULT}" >&2
+    dms_panic__fail_init "${1}"
+  fi
 }
 
-function _start_daemons_rsyslog
+function _start_daemon_changedetector { _default_start_daemon 'changedetector' ; }
+function _start_daemon_amavis         { _default_start_daemon 'amavis'         ; }
+function _start_daemon_clamav         { _default_start_daemon 'clamav'         ; }
+function _start_daemon_cron           { _default_start_daemon 'cron'           ; }
+function _start_daemon_opendkim       { _default_start_daemon 'opendkim'       ; }
+function _start_daemon_opendmarc      { _default_start_daemon 'opendmarc'      ; }
+function _start_daemon_postsrsd       { _default_start_daemon 'postsrsd'       ; }
+function _start_daemon_postfix        { _default_start_daemon 'postfix'        ; }
+function _start_daemon_rsyslog        { _default_start_daemon 'rsyslog'        ; }
+function _start_daemon_update_check   { _default_start_daemon 'update-check'   ; }
+
+function _start_daemon_saslauthd
 {
-  _log 'debug' 'Starting rsyslog'
+  _default_start_daemon "saslauthd_${SASLAUTHD_MECHANISMS}"
-  supervisorctl start rsyslog || dms_panic__fail_init 'rsyslog'
 }
 
-function _start_daemons_saslauthd
+function _start_daemon_postgrey
 {
-  _log 'debug' 'Starting saslauthd'
+  rm -f /var/run/postgrey/postgrey.pid
-  supervisorctl start "saslauthd_${SASLAUTHD_MECHANISMS}" || dms_panic__fail_init 'saslauthd'
+  _default_start_daemon 'postgrey'
 }
 
-function _start_daemons_fail2ban
+function _start_daemon_fail2ban
 {
-  _log 'debug' 'Starting Fail2ban'
   touch /var/log/auth.log
 
   # delete fail2ban.sock that probably was left here after container restart
-  if [[ -e /var/run/fail2ban/fail2ban.sock ]]
+  [[ -e /var/run/fail2ban/fail2ban.sock ]] && rm /var/run/fail2ban/fail2ban.sock
-  then
-    rm /var/run/fail2ban/fail2ban.sock
-  fi
 
-  supervisorctl start fail2ban || dms_panic__fail_init 'Fail2ban'
+  _default_start_daemon 'fail2ban'
 }
 
-function _start_daemons_opendkim
+function _start_daemon_dovecot
 {
-  _log 'debug' 'Starting opendkim'
-  supervisorctl start opendkim || dms_panic__fail_init 'opendkim'
-}
-
-function _start_daemons_opendmarc
-{
-  _log 'debug' 'Starting opendmarc'
-  supervisorctl start opendmarc || dms_panic__fail_init 'opendmarc'
-}
-
-function _start_daemons_postsrsd
-{
-  _log 'debug' 'Starting postsrsd'
-  supervisorctl start postsrsd || dms_panic__fail_init 'postsrsd'
-}
-
-function _start_daemons_postfix
-{
-  _log 'debug' 'Starting postfix'
-  supervisorctl start postfix || dms_panic__fail_init 'postfix'
-}
-
-function _start_daemons_dovecot
-{
-  _log 'debug' 'Starting dovecot services'
-
   if [[ ${ENABLE_POP3} -eq 1 ]]
   then
-    _log 'debug' 'Starting pop3 services'
+    _log 'debug' 'Enabling POP3 services'
-    mv /etc/dovecot/protocols.d/pop3d.protocol.disab \
+    mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol
-      /etc/dovecot/protocols.d/pop3d.protocol
   fi
 
-  if [[ -f /tmp/docker-mailserver/dovecot.cf ]]
+  [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf
-  then
-    cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf
-  fi
 
-  supervisorctl start dovecot || dms_panic__fail_init 'dovecot'
+  _default_start_daemon 'dovecot'
 }
 
-function _start_daemons_fetchmail
+function _start_daemon_fetchmail
 {
-  _log 'debug' 'Preparing fetchmail config'
-  /usr/local/bin/setup-fetchmail
-
   if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]
   then
-    mkdir /etc/fetchmailrc.d/
-    /usr/local/bin/fetchmailrc_split
-
     local COUNTER=0
-    for RC in /etc/fetchmailrc.d/fetchmail-*.rc
-    do
-      COUNTER=$(( COUNTER + 1 ))
-      cat >"/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" << EOF
-[program:fetchmail-${COUNTER}]
-startsecs=0
-autostart=false
-autorestart=true
-stdout_logfile=/var/log/supervisor/%(program_name)s.log
-stderr_logfile=/var/log/supervisor/%(program_name)s.log
-user=fetchmail
-command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid
-EOF
-      chmod 700 "${RC}"
-      chown fetchmail:root "${RC}"
-    done
-
-    supervisorctl reread
-    supervisorctl update
-
-    COUNTER=0
     for _ in /etc/fetchmailrc.d/fetchmail-*.rc
     do
       COUNTER=$(( COUNTER + 1 ))
-      _log 'debug' "Starting fetchmail instance ${COUNTER}"
+      _default_start_daemon "fetchmail-${COUNTER}"
-      supervisorctl start "fetchmail-${COUNTER}" || _panic__fail_init "fetchmail-${COUNTER}"
     done
   else
-    _log 'debug' 'Starting fetchmail'
+    _default_start_daemon 'fetchmail'
-    supervisorctl start fetchmail || dms_panic__fail_init 'fetchmail'
   fi
 }
-
-function _start_daemons_clamav
-{
-  _log 'debug' 'Starting ClamAV'
-  supervisorctl start clamav || dms_panic__fail_init 'ClamAV'
-}
-
-function _start_daemons_postgrey
-{
-  _log 'debug' 'Starting postgrey'
-  rm -f /var/run/postgrey/postgrey.pid
-  supervisorctl start postgrey || dms_panic__fail_init 'postgrey'
-}
-
-function _start_daemons_amavis
-{
-  _log 'debug' 'Starting amavis'
-  supervisorctl start amavis || dms_panic__fail_init 'amavis'
-}
-
-function _start_changedetector
-{
-  _log 'debug' 'Starting changedetector'
-  supervisorctl start changedetector || dms_panic__fail_init 'changedetector'
-}
-
-function _start_daemons_update_check
-{
-  _log 'debug' 'Starting update-check'
-  supervisorctl start update-check || dms_panic__fail_init 'update-check'
-}

target/scripts/startup/setup-stack.sh

@@ -1160,3 +1160,106 @@ function _setup_dnsbl_disable
   postconf -e "postscreen_dnsbl_action = ignore"
   postconf -e "postscreen_dnsbl_sites = "
 }
+
+function _setup_fetchmail
+{
+  _log 'trace' 'Preparing Fetchmail configuration'
+
+  local CONFIGURATION FETCHMAILRC
+
+  CONFIGURATION='/tmp/docker-mailserver/fetchmail.cf'
+  FETCHMAILRC='/etc/fetchmailrc'
+
+  if [[ -f ${CONFIGURATION} ]]
+  then
+    cat /etc/fetchmailrc_general "${CONFIGURATION}" >"${FETCHMAILRC}"
+  else
+    cat /etc/fetchmailrc_general >"${FETCHMAILRC}"
+  fi
+
+  chmod 700 "${FETCHMAILRC}"
+  chown fetchmail:root "${FETCHMAILRC}"
+}
+
+function _setup_fetchmail_parallel
+{
+  _log 'trace' 'Setting up Fetchmail parallel'
+  mkdir /etc/fetchmailrc.d/
+
+  # Split the content of /etc/fetchmailrc into
+  # smaller fetchmailrc files per server [poll] entries. Each
+  # separate fetchmailrc file is stored in /etc/fetchmailrc.d
+  #
+  # The sole purpose for this is to work around what is known
+  # as the Fetchmail IMAP idle issue.
+  function _fetchmailrc_split
+  {
+    local FETCHMAILRC='/etc/fetchmailrc'
+    local FETCHMAILRCD='/etc/fetchmailrc.d'
+    local DEFAULT_FILE="${FETCHMAILRCD}/defaults"
+
+    if [[ ! -r ${FETCHMAILRC} ]]
+    then
+      _log 'warn' "File '${FETCHMAILRC}' not found"
+      return 1
+    fi
+
+    if [[ ! -d ${FETCHMAILRCD} ]]
+    then
+      if ! mkdir "${FETCHMAILRCD}"
+      then
+        _log 'warn' "Unable to create folder '${FETCHMAILRCD}'"
+        return 1
+      fi
+    fi
+
+    local COUNTER=0 SERVER=0
+    while read -r LINE
+    do
+      if [[ ${LINE} =~ poll ]]
+      then
+        # If we read "poll" then we reached a new server definition
+        # We need to create a new file with fetchmail defaults from
+        # /etc/fetcmailrc
+        COUNTER=$(( COUNTER + 1 ))
+        SERVER=1
+        cat "${DEFAULT_FILE}" >"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+        echo "${LINE}" >>"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+      elif [[ ${SERVER} -eq 0 ]]
+      then
+        # We have not yet found "poll". Let's assume we are still reading
+        # the default settings from /etc/fetchmailrc file
+        echo "${LINE}" >>"${DEFAULT_FILE}"
+      else
+        # Just the server settings that need to be added to the specific rc.d file
+        echo "${LINE}" >>"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc"
+      fi
+    # delete commented lines before parsing
+    done < <(sed '/^[[:space:]]*#/d' "${FETCHMAILRC}")
+
+    rm "${DEFAULT_FILE}"
+  }
+
+  _fetchmailrc_split
+
+  local COUNTER=0
+  for RC in /etc/fetchmailrc.d/fetchmail-*.rc
+  do
+    COUNTER=$(( COUNTER + 1 ))
+    cat >"/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" << EOF
+[program:fetchmail-${COUNTER}]
+startsecs=0
+autostart=false
+autorestart=true
+stdout_logfile=/var/log/supervisor/%(program_name)s.log
+stderr_logfile=/var/log/supervisor/%(program_name)s.log
+user=fetchmail
+command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid
+EOF
+    chmod 700 "${RC}"
+    chown fetchmail:root "${RC}"
+  done
+
+  supervisorctl reread
+  supervisorctl update
+}