From a54d7745878c7b3e7958284508aeade7aaa75949 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 27 Mar 2022 09:43:39 +0200 Subject: [PATCH] scripts: refactored `daemon-stack.sh` (#2496) * refactored `daemon-stack.sh` A new method was introduced to uniformaly start daemons and log output accordingly. The methods for daemon start were renamed (plural -> singular), therefore the adjustments in `start-mailserver.sh`. * cleaned Fetchmail setup from `daemon-stack.sh` Not sure why, but the Fetchmail setup was somehow happening in `daemon-stack.sh` - this is not supposed to be the case. I relocated the setup into `setup-stack.sh`, where it belong. * delete old, unnecessary script in `target/bin/` These are unused leftovers from the last commit, that relocated the setup of Fetchmail into `setup.stack.sh`. * corrected changedetector function name * Apply suggestions from code review * adjusted `debug-fetchmail` script It is absolutely fine to source `setup-stack.sh` because sourcing the script does not execute a single function (by desing of the script). This way, we retain functionality. * praise be ShellCheck * added `log.sh` to `debug-fetchmail` as a dependency * final cleanup Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Casper --- target/bin/debug-fetchmail | 7 +- target/bin/fetchmailrc_split | 55 -------- target/bin/setup-fetchmail | 14 -- target/scripts/start-mailserver.sh | 38 +++--- target/scripts/startup/daemons-stack.sh | 162 +++++++----------------- target/scripts/startup/setup-stack.sh | 103 +++++++++++++++ 6 files changed, 175 insertions(+), 204 deletions(-) delete mode 100755 target/bin/fetchmailrc_split delete mode 100755 target/bin/setup-fetchmail diff --git a/target/bin/debug-fetchmail b/target/bin/debug-fetchmail index d5ff3ffb..62d2d6aa 100755 --- a/target/bin/debug-fetchmail +++ b/target/bin/debug-fetchmail @@ -1,6 +1,11 @@ #! /bin/bash -/usr/local/bin/setup-fetchmail +# shellcheck source=../scripts/helpers/log.sh +source /usr/local/bin/helpers/log.sh +# shellcheck source=../scripts/startup/setup-stack.sh +source /usr/local/bin/setup-stack.sh + +_setup_fetchmail su -s /bin/sh -c "/usr/bin/fetchmail \ --verbose \ diff --git a/target/bin/fetchmailrc_split b/target/bin/fetchmailrc_split deleted file mode 100755 index 1f15ad2d..00000000 --- a/target/bin/fetchmailrc_split +++ /dev/null @@ -1,55 +0,0 @@ -#! /bin/bash - -# Description: This script will split the content of /etc/fetchmailrc into -# smaller fetchmailrc files per server [poll] entries. Each -# separate fetchmailrc file is stored in /etc/fetchmailrc.d -# -# The mail purpose for this is to work around what is known -# as the Fetchmail IMAP idle issue. -# - -FETCHMAILRC="/etc/fetchmailrc" -FETCHMAILRCD="/etc/fetchmailrc.d" -DEFAULT_FILE="${FETCHMAILRCD}/defaults" - -if [[ ! -r "${FETCHMAILRC}" ]] -then - _log 'error' "File '${FETCHMAILRC}' not found" - exit 1 -fi - -if [[ ! -d ${FETCHMAILRCD} ]] -then - if ! mkdir "${FETCHMAILRCD}" - then - _log 'error' "Unable to create folder '${FETCHMAILRCD}'" - exit 1 - fi -fi - -COUNTER=0 -SERVER=0 -while read -r LINE -do - if [[ ${LINE} =~ poll ]] - then - # If we read "poll" then we reached a new server definition - # We need to create a new file with fetchmail defaults from - # /etc/fetcmailrc - COUNTER=$((COUNTER+1)) - SERVER=1 - cat "${DEFAULT_FILE}" > "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" - echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" - elif [[ ${SERVER} -eq 0 ]] - then - # We have not yet found "poll". Let's assume we are still reading - # the default settings from /etc/fetchmailrc file - echo "${LINE}" >> "${DEFAULT_FILE}" - else - # Just the server settings that need to be added to the specific rc.d file - echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" - fi -# delete commented lines before parsing -done < <(sed '/^[[:space:]]*#/d' "${FETCHMAILRC}") - -rm "${DEFAULT_FILE}" diff --git a/target/bin/setup-fetchmail b/target/bin/setup-fetchmail deleted file mode 100755 index 7f0897a1..00000000 --- a/target/bin/setup-fetchmail +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -CONF=/tmp/docker-mailserver/fetchmail.cf -RC=/etc/fetchmailrc - -if [[ -f ${CONF} ]] -then - cat /etc/fetchmailrc_general "${CONF}" >"${RC}" -else - cat /etc/fetchmailrc_general >"${RC}" -fi - -chmod 700 "${RC}" -chown fetchmail:root "${RC}" diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 111ced08..06239270 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -170,10 +170,16 @@ function register_functions # needs to come after _setup_postfix_aliases [[ ${SPOOF_PROTECTION} -eq 1 ]] && _register_setup_function '_setup_spoof_protection' + if [[ ${ENABLE_FETCHMAIL} -eq 1 ]] + then + _register_setup_function '_setup_fetchmail' + [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] && _register_setup_function '_setup_fetchmail_parallel' + fi + if [[ ${ENABLE_SRS} -eq 1 ]] then _register_setup_function '_setup_SRS' - _register_start_daemon '_start_daemons_postsrsd' + _register_start_daemon '_start_daemon_postsrsd' fi _register_setup_function '_setup_postfix_access_control' @@ -204,28 +210,28 @@ function register_functions # ? >> Daemons - _register_start_daemon '_start_daemons_cron' - _register_start_daemon '_start_daemons_rsyslog' + _register_start_daemon '_start_daemon_cron' + _register_start_daemon '_start_daemon_rsyslog' - [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemons_dovecot' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemons_update_check' + [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' + [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' # needs to be started before SASLauthd - _register_start_daemon '_start_daemons_opendkim' - _register_start_daemon '_start_daemons_opendmarc' + _register_start_daemon '_start_daemon_opendkim' + _register_start_daemon '_start_daemon_opendmarc' # needs to be started before postfix - [[ ${ENABLE_POSTGREY} -eq 1 ]] && _register_start_daemon '_start_daemons_postgrey' + [[ ${ENABLE_POSTGREY} -eq 1 ]] && _register_start_daemon '_start_daemon_postgrey' - _register_start_daemon '_start_daemons_postfix' + _register_start_daemon '_start_daemon_postfix' # needs to be started after postfix - [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && _register_start_daemon '_start_daemons_saslauthd' - [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && _register_start_daemon '_start_daemons_fail2ban' - [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && _register_start_daemon '_start_daemons_fetchmail' - [[ ${ENABLE_CLAMAV} -eq 1 ]] && _register_start_daemon '_start_daemons_clamav' - [[ ${ENABLE_LDAP} -eq 0 ]] && _register_start_daemon '_start_changedetector' - [[ ${ENABLE_AMAVIS} -eq 1 ]] && _register_start_daemon '_start_daemons_amavis' + [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && _register_start_daemon '_start_daemon_saslauthd' + [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && _register_start_daemon '_start_daemon_fail2ban' + [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && _register_start_daemon '_start_daemon_fetchmail' + [[ ${ENABLE_CLAMAV} -eq 1 ]] && _register_start_daemon '_start_daemon_clamav' + [[ ${ENABLE_LDAP} -eq 0 ]] && _register_start_daemon '_start_daemon_changedetector' + [[ ${ENABLE_AMAVIS} -eq 1 ]] && _register_start_daemon '_start_daemon_amavis' } function _register_start_daemon @@ -275,7 +281,7 @@ setup [[ ${LOG_LEVEL} =~ (debug|trace) ]] && print-environment fix start_misc -start_daemons +_start_daemons # marker to check, if container was restarted date >/CONTAINER_START diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index 035b38a9..cdfcbd82 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -1,160 +1,86 @@ #! /bin/bash -function start_daemons +function _start_daemons { - _log 'info' 'Starting daemons & mail server' - for FUNC in "${DAEMONS_START[@]}" + _log 'info' 'Starting daemons' + + for FUNCTION in "${DAEMONS_START[@]}" do - ${FUNC} + ${FUNCTION} done } -function _start_daemons_cron +function _default_start_daemon { - _log 'debug' 'Starting cron' - supervisorctl start cron || dms_panic__fail_init 'cron' + _log 'debug' "Starting ${1:?}" + + local RESULT + RESULT="$(supervisorctl start "${1}" 2>&1)" + + # shellcheck disable=SC2181 + if [[ ${?} -ne 0 ]] + then + echo "${RESULT}" >&2 + dms_panic__fail_init "${1}" + fi } -function _start_daemons_rsyslog +function _start_daemon_changedetector { _default_start_daemon 'changedetector' ; } +function _start_daemon_amavis { _default_start_daemon 'amavis' ; } +function _start_daemon_clamav { _default_start_daemon 'clamav' ; } +function _start_daemon_cron { _default_start_daemon 'cron' ; } +function _start_daemon_opendkim { _default_start_daemon 'opendkim' ; } +function _start_daemon_opendmarc { _default_start_daemon 'opendmarc' ; } +function _start_daemon_postsrsd { _default_start_daemon 'postsrsd' ; } +function _start_daemon_postfix { _default_start_daemon 'postfix' ; } +function _start_daemon_rsyslog { _default_start_daemon 'rsyslog' ; } +function _start_daemon_update_check { _default_start_daemon 'update-check' ; } + +function _start_daemon_saslauthd { - _log 'debug' 'Starting rsyslog' - supervisorctl start rsyslog || dms_panic__fail_init 'rsyslog' + _default_start_daemon "saslauthd_${SASLAUTHD_MECHANISMS}" } -function _start_daemons_saslauthd +function _start_daemon_postgrey { - _log 'debug' 'Starting saslauthd' - supervisorctl start "saslauthd_${SASLAUTHD_MECHANISMS}" || dms_panic__fail_init 'saslauthd' + rm -f /var/run/postgrey/postgrey.pid + _default_start_daemon 'postgrey' } -function _start_daemons_fail2ban +function _start_daemon_fail2ban { - _log 'debug' 'Starting Fail2ban' touch /var/log/auth.log # delete fail2ban.sock that probably was left here after container restart - if [[ -e /var/run/fail2ban/fail2ban.sock ]] - then - rm /var/run/fail2ban/fail2ban.sock - fi + [[ -e /var/run/fail2ban/fail2ban.sock ]] && rm /var/run/fail2ban/fail2ban.sock - supervisorctl start fail2ban || dms_panic__fail_init 'Fail2ban' + _default_start_daemon 'fail2ban' } -function _start_daemons_opendkim +function _start_daemon_dovecot { - _log 'debug' 'Starting opendkim' - supervisorctl start opendkim || dms_panic__fail_init 'opendkim' -} - -function _start_daemons_opendmarc -{ - _log 'debug' 'Starting opendmarc' - supervisorctl start opendmarc || dms_panic__fail_init 'opendmarc' -} - -function _start_daemons_postsrsd -{ - _log 'debug' 'Starting postsrsd' - supervisorctl start postsrsd || dms_panic__fail_init 'postsrsd' -} - -function _start_daemons_postfix -{ - _log 'debug' 'Starting postfix' - supervisorctl start postfix || dms_panic__fail_init 'postfix' -} - -function _start_daemons_dovecot -{ - _log 'debug' 'Starting dovecot services' - if [[ ${ENABLE_POP3} -eq 1 ]] then - _log 'debug' 'Starting pop3 services' - mv /etc/dovecot/protocols.d/pop3d.protocol.disab \ - /etc/dovecot/protocols.d/pop3d.protocol + _log 'debug' 'Enabling POP3 services' + mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol fi - if [[ -f /tmp/docker-mailserver/dovecot.cf ]] - then - cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf - fi + [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf - supervisorctl start dovecot || dms_panic__fail_init 'dovecot' + _default_start_daemon 'dovecot' } -function _start_daemons_fetchmail +function _start_daemon_fetchmail { - _log 'debug' 'Preparing fetchmail config' - /usr/local/bin/setup-fetchmail - if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] then - mkdir /etc/fetchmailrc.d/ - /usr/local/bin/fetchmailrc_split - local COUNTER=0 - for RC in /etc/fetchmailrc.d/fetchmail-*.rc - do - COUNTER=$(( COUNTER + 1 )) - cat >"/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" << EOF -[program:fetchmail-${COUNTER}] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -user=fetchmail -command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid -EOF - chmod 700 "${RC}" - chown fetchmail:root "${RC}" - done - - supervisorctl reread - supervisorctl update - - COUNTER=0 for _ in /etc/fetchmailrc.d/fetchmail-*.rc do COUNTER=$(( COUNTER + 1 )) - _log 'debug' "Starting fetchmail instance ${COUNTER}" - supervisorctl start "fetchmail-${COUNTER}" || _panic__fail_init "fetchmail-${COUNTER}" + _default_start_daemon "fetchmail-${COUNTER}" done else - _log 'debug' 'Starting fetchmail' - supervisorctl start fetchmail || dms_panic__fail_init 'fetchmail' + _default_start_daemon 'fetchmail' fi } - -function _start_daemons_clamav -{ - _log 'debug' 'Starting ClamAV' - supervisorctl start clamav || dms_panic__fail_init 'ClamAV' -} - -function _start_daemons_postgrey -{ - _log 'debug' 'Starting postgrey' - rm -f /var/run/postgrey/postgrey.pid - supervisorctl start postgrey || dms_panic__fail_init 'postgrey' -} - -function _start_daemons_amavis -{ - _log 'debug' 'Starting amavis' - supervisorctl start amavis || dms_panic__fail_init 'amavis' -} - -function _start_changedetector -{ - _log 'debug' 'Starting changedetector' - supervisorctl start changedetector || dms_panic__fail_init 'changedetector' -} - -function _start_daemons_update_check -{ - _log 'debug' 'Starting update-check' - supervisorctl start update-check || dms_panic__fail_init 'update-check' -} diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 49c273fb..6f7a3763 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -1160,3 +1160,106 @@ function _setup_dnsbl_disable postconf -e "postscreen_dnsbl_action = ignore" postconf -e "postscreen_dnsbl_sites = " } + +function _setup_fetchmail +{ + _log 'trace' 'Preparing Fetchmail configuration' + + local CONFIGURATION FETCHMAILRC + + CONFIGURATION='/tmp/docker-mailserver/fetchmail.cf' + FETCHMAILRC='/etc/fetchmailrc' + + if [[ -f ${CONFIGURATION} ]] + then + cat /etc/fetchmailrc_general "${CONFIGURATION}" >"${FETCHMAILRC}" + else + cat /etc/fetchmailrc_general >"${FETCHMAILRC}" + fi + + chmod 700 "${FETCHMAILRC}" + chown fetchmail:root "${FETCHMAILRC}" +} + +function _setup_fetchmail_parallel +{ + _log 'trace' 'Setting up Fetchmail parallel' + mkdir /etc/fetchmailrc.d/ + + # Split the content of /etc/fetchmailrc into + # smaller fetchmailrc files per server [poll] entries. Each + # separate fetchmailrc file is stored in /etc/fetchmailrc.d + # + # The sole purpose for this is to work around what is known + # as the Fetchmail IMAP idle issue. + function _fetchmailrc_split + { + local FETCHMAILRC='/etc/fetchmailrc' + local FETCHMAILRCD='/etc/fetchmailrc.d' + local DEFAULT_FILE="${FETCHMAILRCD}/defaults" + + if [[ ! -r ${FETCHMAILRC} ]] + then + _log 'warn' "File '${FETCHMAILRC}' not found" + return 1 + fi + + if [[ ! -d ${FETCHMAILRCD} ]] + then + if ! mkdir "${FETCHMAILRCD}" + then + _log 'warn' "Unable to create folder '${FETCHMAILRCD}'" + return 1 + fi + fi + + local COUNTER=0 SERVER=0 + while read -r LINE + do + if [[ ${LINE} =~ poll ]] + then + # If we read "poll" then we reached a new server definition + # We need to create a new file with fetchmail defaults from + # /etc/fetcmailrc + COUNTER=$(( COUNTER + 1 )) + SERVER=1 + cat "${DEFAULT_FILE}" >"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + echo "${LINE}" >>"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + elif [[ ${SERVER} -eq 0 ]] + then + # We have not yet found "poll". Let's assume we are still reading + # the default settings from /etc/fetchmailrc file + echo "${LINE}" >>"${DEFAULT_FILE}" + else + # Just the server settings that need to be added to the specific rc.d file + echo "${LINE}" >>"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + fi + # delete commented lines before parsing + done < <(sed '/^[[:space:]]*#/d' "${FETCHMAILRC}") + + rm "${DEFAULT_FILE}" + } + + _fetchmailrc_split + + local COUNTER=0 + for RC in /etc/fetchmailrc.d/fetchmail-*.rc + do + COUNTER=$(( COUNTER + 1 )) + cat >"/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" << EOF +[program:fetchmail-${COUNTER}] +startsecs=0 +autostart=false +autorestart=true +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +user=fetchmail +command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid +EOF + chmod 700 "${RC}" + chown fetchmail:root "${RC}" + done + + supervisorctl reread + supervisorctl update +}