chore(housekeeping): Create helpers/change-detection.sh (#2610)

* chore: Extract change-detection method to it's own helper

This doesn't really belong in `helpers/ssl.sh`. Moving to it's own helper script.

* chore: Co-locate related change-detection method from container startup

It seems relevant to migrate the related support during startup for the change detection feature into this helper.

I opted to move the call from `start-mailserver.sh` into the `_setup` call at the end for a more explicit/visible location.

* chore: Move `CHKSUM_FILE` into `helpers/change-detection.sh`

It belongs there, not in `helpers/index.sh`.

* chore: Revise inline documentation

* tests(fix): Ensure correct functionality

Presently `test/test_helper.bats` is using it's own  `CHKSUM_FILE` instead of sourcing the var for the filepath.

`test_helper/common.bash` was calling a method to check for changes, but this helper may not correctly detect letsencrypt related changes as these are not ENV rely on, but global vars handled by `helpers/dns.sh`, so that should be run first like it is for `check-for-changes.sh`.

* tests(chore): Use `CHKSUM_FILE` var from helper

* chore: `addmailuser` should use `CHKSUM_FILE` var

* chore: Update `check-for-changes.sh` log message with correct path

target/bin/addmailuser

@@ -63,7 +63,8 @@ USER="${FULL_EMAIL%@*}"
 DOMAIN="${FULL_EMAIL#*@}"
 
 # Tests fail if the creation of /var/mail/${DOMAIN}/${USER} doesn't happen fast enough after addmailuser executes (check-for-changes.sh race-condition)
-if [[ -e /tmp/docker-mailserver-config-chksum ]] # Prevent infinite loop in tests like "checking accounts: user3 should have been added to /tmp/docker-mailserver/postfix-accounts.cf even when that file does not exist"
+# Prevent infinite loop in tests like "checking accounts: user3 should have been added to /tmp/docker-mailserver/postfix-accounts.cf even when that file does not exist"
+if [[ -e ${CHKSUM_FILE} ]]
 then
   while [[ ! -d "/var/mail/${DOMAIN}/${USER}" ]]
   do

target/scripts/check-for-changes.sh

@@ -36,7 +36,7 @@ fi
 # verify checksum file exists; must be prepared by start-mailserver.sh
 if [[ ! -f ${CHKSUM_FILE} ]]
 then
-  _exit_with_error "'/tmp/docker-mailserver/${CHKSUM_FILE}' is missing" 0
+  _exit_with_error "'${CHKSUM_FILE}' is missing" 0
 fi
 
 REGEX_NEVER_MATCH="(?\!)"

target/scripts/helpers/change-detection.sh

@@ -0,0 +1,78 @@
+#! /bin/bash
+
+# This helper supports the changedetector service. Used by:
+# - check-for-changes.sh
+# - test/test_helper/common.bash:wait_for_changes_to_be_detected_in_container()
+# - test/test_helper.bats
+# - start-mailserver.sh --> setup-stack.sh (to initialize the CHKSUM_FILE state)
+
+# Global checksum file used to track when monitored files have changed in content:
+# shellcheck disable=SC2034
+CHKSUM_FILE=/tmp/docker-mailserver-config-chksum
+
+# Once container startup scripts complete, take a snapshot of
+# the config state via storing a list of files content hashes.
+function _prepare_for_change_detection
+{
+  _log 'debug' 'Setting up configuration checksum file'
+
+  if [[ -d /tmp/docker-mailserver ]]
+  then
+    _log 'trace' "Creating '${CHKSUM_FILE}'"
+    _monitored_files_checksums >"${CHKSUM_FILE}"
+  else
+    # We could just skip the file, but perhaps config can be added later?
+    # If so it must be processed by the check for changes script
+    _log 'trace' "Creating empty '${CHKSUM_FILE}' (no config)"
+    touch "${CHKSUM_FILE}"
+  fi
+}
+
+# Returns a list of changed files, each line is a value pair of:
+# <SHA-512 content hash> <changed file path>
+function _monitored_files_checksums
+{
+  local DMS_DIR=/tmp/docker-mailserver
+  [[ -d ${DMS_DIR} ]] || return 1
+
+  # If a wildcard path pattern (or an empty ENV) would yield an invalid path
+  # or no results, `shopt -s nullglob` prevents it from being added.
+  shopt -s nullglob
+  declare -a STAGING_FILES CHANGED_FILES
+
+  STAGING_FILES=(
+    "${DMS_DIR}/postfix-accounts.cf"
+    "${DMS_DIR}/postfix-virtual.cf"
+    "${DMS_DIR}/postfix-aliases.cf"
+    "${DMS_DIR}/dovecot-quotas.cf"
+    "${DMS_DIR}/dovecot-masters.cf"
+  )
+
+  if [[ ${SSL_TYPE:-} == 'manual' ]]
+  then
+    # When using "manual" as the SSL type,
+    # the following variables may contain the certificate files
+    STAGING_FILES+=(
+      "${SSL_CERT_PATH:-}"
+      "${SSL_KEY_PATH:-}"
+      "${SSL_ALT_CERT_PATH:-}"
+      "${SSL_ALT_KEY_PATH:-}"
+    )
+  elif [[ ${SSL_TYPE:-} == 'letsencrypt' ]]
+  then
+    # React to any cert changes within the following LetsEncrypt locations:
+    STAGING_FILES+=(
+      /etc/letsencrypt/acme.json
+      /etc/letsencrypt/live/"${SSL_DOMAIN}"/*.pem
+      /etc/letsencrypt/live/"${HOSTNAME}"/*.pem
+      /etc/letsencrypt/live/"${DOMAINNAME}"/*.pem
+    )
+  fi
+
+  for FILE in "${STAGING_FILES[@]}"
+  do
+    [[ -f "${FILE}" ]] && CHANGED_FILES+=("${FILE}")
+  done
+
+  sha512sum -- "${CHANGED_FILES[@]}"
+}

target/scripts/helpers/index.sh

@@ -3,23 +3,13 @@
 # shellcheck source-path=target/scripts/helpers
 # This file serves as a single import for all helpers
 
-# Global checksum file mainly needed for the changedetector.
-# Used in the folling scripts:
-#
-# - ../check-for-changes.sh
-# - ../start-mailserver.sh
-# - ../startup/setup-stack.sh
-# - ../../../test/test_helper/common.bash
-#
-# shellcheck disable=SC2034
-CHKSUM_FILE=/tmp/docker-mailserver-config-chksum
-
 function _import_scripts
 {
   local PATH_TO_SCRIPTS='/usr/local/bin/helpers'
 
   source "${PATH_TO_SCRIPTS}/accounts.sh"
   source "${PATH_TO_SCRIPTS}/aliases.sh"
+  source "${PATH_TO_SCRIPTS}/change-detection.sh"
   source "${PATH_TO_SCRIPTS}/dns.sh"
   source "${PATH_TO_SCRIPTS}/error.sh"
   source "${PATH_TO_SCRIPTS}/lock.sh"

target/scripts/helpers/ssl.sh

@@ -465,52 +465,3 @@ function _extract_certs_from_acme
 function _strip_wildcard_prefix {
   [[ ${1} == "*."* ]] && echo "${1:2}" || echo "${1}"
 }
-
-# Compute checksums of monitored files,
-# returned output on `stdout`: hash + filepath tuple on each line
-function _monitored_files_checksums
-{
-  local DMS_DIR=/tmp/docker-mailserver
-  [[ -d ${DMS_DIR} ]] || return 1
-
-  # If a wildcard path pattern (or an empty ENV) would yield an invalid path
-  # or no results, `shopt -s nullglob` prevents it from being added.
-  shopt -s nullglob
-  declare -a STAGING_FILES CHANGED_FILES
-
-  STAGING_FILES=(
-    "${DMS_DIR}/postfix-accounts.cf"
-    "${DMS_DIR}/postfix-virtual.cf"
-    "${DMS_DIR}/postfix-aliases.cf"
-    "${DMS_DIR}/dovecot-quotas.cf"
-    "${DMS_DIR}/dovecot-masters.cf"
-  )
-
-  if [[ ${SSL_TYPE:-} == 'manual' ]]
-  then
-    # When using "manual" as the SSL type,
-    # the following variables may contain the certificate files
-    STAGING_FILES+=(
-      "${SSL_CERT_PATH:-}"
-      "${SSL_KEY_PATH:-}"
-      "${SSL_ALT_CERT_PATH:-}"
-      "${SSL_ALT_KEY_PATH:-}"
-    )
-  elif [[ ${SSL_TYPE:-} == 'letsencrypt' ]]
-  then
-    # React to any cert changes within the following Let'sEncrypt locations:
-    STAGING_FILES+=(
-      /etc/letsencrypt/acme.json
-      /etc/letsencrypt/live/"${SSL_DOMAIN}"/*.pem
-      /etc/letsencrypt/live/"${HOSTNAME}"/*.pem
-      /etc/letsencrypt/live/"${DOMAINNAME}"/*.pem
-    )
-  fi
-
-  for FILE in "${STAGING_FILES[@]}"
-  do
-    [[ -f "${FILE}" ]] && CHANGED_FILES+=("${FILE}")
-  done
-
-  sha512sum -- "${CHANGED_FILES[@]}"
-}

target/scripts/start-mailserver.sh

@@ -245,9 +245,6 @@ function _register_functions
   _register_setup_function '_setup_logwatch'
   _register_setup_function '_setup_user_patches'
 
-  # needs to come last as configuration files are modified in-place
-  _register_setup_function '_setup_chksum_file'
-
   # ? >> Fixes
 
   _register_fix_function '_fix_var_mail_permissions'

target/scripts/startup/setup-stack.sh

@@ -7,6 +7,9 @@ function _setup
   do
     ${FUNC}
   done
+
+  # All startup modifications to configs should have taken place before calling this:
+  _prepare_for_change_detection
 }
 
 function _setup_supervisor
@@ -77,22 +80,6 @@ function _setup_file_permissions
   chmod 640 /var/log/mail/freshclam.log
 }
 
-function _setup_chksum_file
-{
-  _log 'debug' 'Setting up configuration checksum file'
-
-  if [[ -d /tmp/docker-mailserver ]]
-  then
-    _log 'trace' "Creating '${CHKSUM_FILE}'"
-    _monitored_files_checksums >"${CHKSUM_FILE}"
-  else
-    # We could just skip the file, but perhaps config can be added later?
-    # If so it must be processed by the check for changes script
-    _log 'trace' "Creating empty '${CHKSUM_FILE}' (no config)"
-    touch "${CHKSUM_FILE}"
-  fi
-}
-
 function _setup_mailname
 {
   _log 'debug' "Setting up mailname and creating '/etc/mailname'"

test/test_helper.bats

@@ -170,8 +170,9 @@ load 'test_helper/common'
 
   teardown() { docker rm -f "${CONTAINER_NAME}"; }
 
-  # wait for the initial checksum detection to complete
+  # wait for the initial checksum file to be created
-  repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /tmp/docker-mailserver-config-chksum
+  # shellcheck disable=SC2016
+  repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" bash -c 'source /usr/local/bin/helpers/index.sh; test -e "${CHKSUM_FILE}"'
 
   # there should be no changes in the beginning
   TEST_TIMEOUT_IN_SECONDS=0 wait_for_changes_to_be_detected_in_container "${CONTAINER_NAME}"
@@ -195,8 +196,9 @@ load 'test_helper/common'
 
   teardown() { docker rm -f "${CONTAINER_NAME}"; }
 
-  # wait for the initial checksum detection to complete
+  # wait for the initial checksum file to be created
-  repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /tmp/docker-mailserver-config-chksum
+  # shellcheck disable=SC2016
+  repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" bash -c 'source /usr/local/bin/helpers/index.sh; test -e "${CHKSUM_FILE}"'
 
   # trigger some change
   docker exec "${CONTAINER_NAME}" /bin/sh -c "addmailuser auser3@mail.my-domain.com mypassword"

test/test_helper/common.bash

@@ -170,7 +170,7 @@ function wait_for_changes_to_be_detected_in_container() {
   local TIMEOUT=${TEST_TIMEOUT_IN_SECONDS}
 
   # shellcheck disable=SC2016
-  repeat_in_container_until_success_or_timeout "${TIMEOUT}" "${CONTAINER_NAME}" bash -c 'source /usr/local/bin/helpers/index.sh; cmp --silent -- <(_monitored_files_checksums) "${CHKSUM_FILE}" >/dev/null'
+  repeat_in_container_until_success_or_timeout "${TIMEOUT}" "${CONTAINER_NAME}" bash -c 'source /usr/local/bin/helpers/index.sh; _obtain_hostname_and_domainname; cmp --silent -- <(_monitored_files_checksums) "${CHKSUM_FILE}" >/dev/null'
 }
 
 function wait_for_empty_mail_queue_in_container() {