chore: Add debug group (packages.sh) + more resilient rspamd setup (#3578)

target/scripts/build/packages.sh

@@ -43,10 +43,6 @@ function _install_postfix() {
 function _install_packages() {
   _log 'debug' 'Installing all packages now'
 
-  declare -a ANTI_VIRUS_SPAM_PACKAGES
-  declare -a CODECS_PACKAGES MISCELLANEOUS_PACKAGES
-  declare -a POSTFIX_PACKAGES MAIL_PROGRAMS_PACKAGES
-
   ANTI_VIRUS_SPAM_PACKAGES=(
     amavisd-new clamav clamav-daemon
     pyzor razor spamassassin
@@ -62,14 +58,13 @@ function _install_packages() {
   )
 
   MISCELLANEOUS_PACKAGES=(
-    apt-transport-https bind9-dnsutils binutils bsd-mailx
+    apt-transport-https binutils bsd-mailx
     ca-certificates curl dbconfig-no-thanks
-    dumb-init ed gnupg iproute2 iputils-ping
-    libdate-manip-perl libldap-common
-    libmail-spf-perl libnet-dns-perl
-    locales logwatch netcat-openbsd
-    nftables rsyslog supervisor
-    uuid whois
+    dumb-init gnupg iproute2 libdate-manip-perl
+    libldap-common libmail-spf-perl
+    libnet-dns-perl locales logwatch
+    netcat-openbsd nftables rsyslog
+    supervisor uuid whois
   )
 
   POSTFIX_PACKAGES=(
@@ -82,12 +77,19 @@ function _install_packages() {
     opendmarc libsasl2-modules sasl2-bin
   )
 
+  # `bind9-dnsutils` provides the `dig` command
+  # `iputils-ping` provides the `ping` command
+  DEBUG_PACKAGES=(
+    bind9-dnsutils iputils-ping less nano
+  )
+
   apt-get "${QUIET}" --no-install-recommends install \
     "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \
     "${CODECS_PACKAGES[@]}" \
     "${MISCELLANEOUS_PACKAGES[@]}" \
     "${POSTFIX_PACKAGES[@]}" \
-    "${MAIL_PROGRAMS_PACKAGES[@]}"
+    "${MAIL_PROGRAMS_PACKAGES[@]}" \
+    "${DEBUG_PACKAGES[@]}"
 }
 
 function _install_dovecot() {

target/scripts/helpers/utils.sh

@@ -127,9 +127,17 @@ function _replace_by_env_in_file() {
 function _env_var_expect_zero_or_one() {
   local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_zero_or_one}
 
-  [[ ${!ENV_VAR_NAME} =~ ^(0|1)$ ]] && return 0
-  _log 'warn' "The value of '${ENV_VAR_NAME}' is not zero or one ('${!ENV_VAR_NAME}'), but was expected to be"
+  if [[ ! -v ${ENV_VAR_NAME} ]]; then
+    _log 'warn' "'${ENV_VAR_NAME}' is not set, but was expected to be"
     return 1
+  fi
+
+  if [[ ! ${!ENV_VAR_NAME} =~ ^(0|1)$ ]]; then
+    _log 'warn' "The value of '${ENV_VAR_NAME}' (= '${!ENV_VAR_NAME}') is not 0 or 1, but was expected to be"
+    return 1
+  fi
+
+  return 0
 }
 
 # Check if an environment variable's value is an integer.

target/scripts/startup/setup.d/security/rspamd.sh

@@ -43,6 +43,8 @@ function __rspamd__helper__enable_disable_module() {
   local LOCAL_OR_OVERRIDE=${3:-local}
   local MESSAGE='Enabling'
 
+  readonly MODULE ENABLE_MODULE LOCAL_OR_OVERRIDE
+
   if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]]; then
     __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not"
     return 1
@@ -64,10 +66,12 @@ EOF
 function __rspamd__run_early_setup_and_checks() {
   # Note: Variables not marked with `local` are
   # used in other functions as well.
-  RSPAMD_LOCAL_D='/etc/rspamd/local.d'
-  RSPAMD_OVERRIDE_D='/etc/rspamd/override.d'
-  RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd'
+  readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d'
+  readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d'
+  readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd'
+
   local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/"
+  readonly RSPAMD_DMS_OVERRIDE_D
 
   mkdir -p /var/lib/rspamd/
   : >/var/lib/rspamd/stats.ucl
@@ -77,7 +81,7 @@ function __rspamd__run_early_setup_and_checks() {
     if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then
       ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}"
     else
-      __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty? not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'"
+      __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'"
     fi
   fi
 
@@ -195,6 +199,7 @@ function __rspamd__setup_default_modules() {
     metric_exporter
   )
 
+  readonly -a DISABLE_MODULES
   local MODULE
   for MODULE in "${DISABLE_MODULES[@]}"; do
     __rspamd__helper__enable_disable_module "${MODULE}" 'false'
@@ -211,6 +216,7 @@ function __rspamd__setup_learning() {
     __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham'
 
     local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe'
+    readonly SIEVE_PIPE_BIN_DIR
     ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc"
 
     sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf
@@ -264,6 +270,7 @@ function __rspamd__setup_greylisting() {
 # succeeds.
 function __rspamd__setup_hfilter_group() {
   local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf"
+  readonly MODULE_FILE
   if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then
     __rspamd__log 'debug' 'Hfilter (group) module is enabled'
     # Check if we received a number first
@@ -284,6 +291,7 @@ function __rspamd__setup_hfilter_group() {
 
 function __rspamd__setup_check_authenticated() {
   local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf"
+  readonly MODULE_FILE
   if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \
   && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]]
   then
@@ -320,8 +328,10 @@ function __rspamd__handle_user_modules_adjustments() {
     local VALUE=${4:?Value belonging to an option must be provided}
     # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty)
     VALUE=${VALUE% }
-
     local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}"
+
+    readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE
+
     [[ -f ${FILE} ]] || touch "${FILE}"
 
     if grep -q -E "${OPTION}.*=.*" "${FILE}"; then
@@ -335,6 +345,7 @@ function __rspamd__handle_user_modules_adjustments() {
 
   local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf"
   local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf"
+  readonly RSPAMD_CUSTOM_COMMANDS_FILE RSPAMD_CUSTOM_COMMANDS_FILE_OLD
 
   # We check for usage of the previous location of the commands file.
   # This can be removed after the release of v14.0.0.
@@ -347,6 +358,7 @@ function __rspamd__handle_user_modules_adjustments() {
   if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then
     __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it"
 
+    local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3
     while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do
       case "${COMMAND}" in
         ('disable-module')

test/tests/parallel/set3/scripts/helper_functions.bats

@@ -35,7 +35,11 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/
 
   run _env_var_expect_zero_or_one TWO
   assert_failure
-  assert_output --partial "The value of 'TWO' is not zero or one ('2'), but was expected to be"
+  assert_output --partial "The value of 'TWO' (= '2') is not 0 or 1, but was expected to be"
+
+  run _env_var_expect_zero_or_one UNSET
+  assert_failure
+  assert_output --partial "'UNSET' is not set, but was expected to be"
 
   run _env_var_expect_zero_or_one
   assert_failure