ci: enhance build process (#2755)

This new script is a clean way of handling the installation of packages.
I think the huge `RUN` command in `Dockerfile` was hard to read and
maintain.

Using a script is a non-issue, as the image is rebuilt whenever the
script is touched.

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>

.github/workflows/generic_publish.yml

@@ -72,7 +72,7 @@ jobs:
         run: echo "::set-output name=version::$(<VERSION)"
 
       - name: 'Build and publish images'
-        uses: docker/build-push-action@v3.1
+        uses: docker/build-push-action@v3.1.1
         with:
           context: .
           build-args: |

Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker.io/docker/dockerfile:1
+
 # This Dockerfile provides two stages: stage-base and stage-final
 # This is in preparation for more granular stages (eg ClamAV and Fail2Ban split into their own)
 
@@ -7,73 +9,23 @@
 
 FROM docker.io/debian:11-slim AS stage-base
 
+ARG LOG_LEVEL=trace
 ARG DEBIAN_FRONTEND=noninteractive
 
-ARG FAIL2BAN_DEB_URL=https://github.com/fail2ban/fail2ban/releases/download/0.11.2/fail2ban_0.11.2-1.upstream1_all.deb
-ARG FAIL2BAN_DEB_ASC_URL=${FAIL2BAN_DEB_URL}.asc
-ARG FAIL2BAN_GPG_PUBLIC_KEY_ID=0x683BF1BEBD0A882C
-ARG FAIL2BAN_GPG_PUBLIC_KEY_SERVER=hkps://keyserver.ubuntu.com
-ARG FAIL2BAN_GPG_FINGERPRINT="8738 559E 26F6 71DF 9E2C  6D9E 683B F1BE BD0A 882C"
-
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # -----------------------------------------------
 # --- Install Basic Software --------------------
 # -----------------------------------------------
 
-RUN \
+COPY target/scripts/build/* /build/
-  apt-get -qq update && \
-  apt-get -qq install apt-utils 2>/dev/null && \
-  apt-get -qq dist-upgrade && \
-  echo "applying workaround for ubuntu/postfix bug described in https://github.com/docker-mailserver/docker-mailserver/issues/2023#issuecomment-855326403" && \
-  mv /bin/hostname{,.bak} && \
-  echo "echo docker-mailserver.invalid" > /bin/hostname && \
-  chmod +x /bin/hostname && \
-  apt-get -qq install postfix && \
-  mv /bin/hostname{.bak,} && \
-  apt-get -qq --no-install-recommends install \
-  # A - D
-  altermime amavisd-new apt-transport-https arj binutils bzip2 bsd-mailx \
-  ca-certificates cabextract clamav clamav-daemon cpio curl \
-  dbconfig-no-thanks dovecot-core dovecot-fts-xapian dovecot-imapd \
-  dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d \
-  dovecot-sieve dovecot-solr dumb-init \
-  # E - O
-  ed fetchmail file gamin gnupg gzip iproute2 \
-  locales logwatch lhasa libdate-manip-perl libldap-common liblz4-tool \
-  libmail-spf-perl libnet-dns-perl libsasl2-modules lrzip lzop \
-  netcat-openbsd nftables nomarch opendkim opendkim-tools opendmarc \
-  # P - Z
-  pax pflogsumm postgrey p7zip-full postfix-ldap postfix-pcre \
-  postfix-policyd-spf-python postsrsd pyzor \
-  razor rpm2cpio rsyslog sasl2-bin spamassassin supervisor \
-  unrar-free unzip uuid whois xz-utils && \
-  # Fail2Ban
-  gpg --keyserver ${FAIL2BAN_GPG_PUBLIC_KEY_SERVER} \
-    --recv-keys ${FAIL2BAN_GPG_PUBLIC_KEY_ID} 2>&1 && \
-  curl -Lkso fail2ban.deb ${FAIL2BAN_DEB_URL} && \
-  curl -Lkso fail2ban.deb.asc ${FAIL2BAN_DEB_ASC_URL} && \
-  FINGERPRINT=$(LANG=C gpg --verify \
-  fail2ban.deb.asc fail2ban.deb 2>&1 \
-    | sed -n 's#Primary key fingerprint: \(.*\)#\1#p') && \
-  if [[ -z ${FINGERPRINT} ]]; then \
-    echo "ERROR: Invalid GPG signature!" >&2; exit 1; fi && \
-  if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]]; then \
-    echo "ERROR: Wrong GPG fingerprint!" >&2; exit 1; fi && \
-  dpkg -i fail2ban.deb 2>&1 && \
-  rm fail2ban.deb fail2ban.deb.asc && \
-  # cleanup
-  apt-get -qq autoremove && \
-  apt-get -qq autoclean && \
-  apt-get -qq clean && \
-  rm -rf /var/lib/apt/lists/* && \
-  c_rehash 2>&1
-
 COPY target/scripts/helpers/log.sh /usr/local/bin/helpers/log.sh
-COPY target/bin/sedfile /usr/local/bin/sedfile
+RUN /bin/bash /build/packages.sh
 
+COPY target/bin/sedfile /usr/local/bin/sedfile
 RUN chmod +x /usr/local/bin/sedfile
 
+
 # -----------------------------------------------
 # --- ClamAV & FeshClam -------------------------
 # -----------------------------------------------
@@ -81,12 +33,15 @@ RUN chmod +x /usr/local/bin/sedfile
 RUN \
   echo '0 */6 * * * clamav /usr/bin/freshclam --quiet' >/etc/cron.d/clamav-freshclam && \
   chmod 644 /etc/clamav/freshclam.conf && \
-  freshclam && \
   sedfile -i 's/Foreground false/Foreground true/g' /etc/clamav/clamd.conf && \
   mkdir /var/run/clamav && \
   chown -R clamav:root /var/run/clamav && \
   rm -rf /var/log/clamav/
 
+# Copy over latest DB updates from official ClamAV image. Better than running `freshclam` (which requires extra RAM during build)
+# hadolint ignore=DL3021
+COPY --link --from=docker.io/clamav/clamav:latest /var/lib/clamav /var/lib/clamav
+
 # -----------------------------------------------
 # --- Dovecot -----------------------------------
 # -----------------------------------------------

Makefile

@@ -11,7 +11,10 @@ VCS_VERSION  = $(shell cat VERSION)
 all: lint build backup generate-accounts tests clean
 
 build:
-	docker build -t $(NAME) . --build-arg VCS_VERSION=$(VCS_VERSION) --build-arg VCS_REVISION=$(VCS_REVISION)
+	@ DOCKER_BUILDKIT=1 docker build --tag $(NAME) \
+		--build-arg VCS_VERSION=$(VCS_VERSION) \
+		--build-arg VCS_REVISION=$(VCS_REVISION) \
+		.
 
 backup:
 # if backup directory exist, clean hasn't been called, therefore

target/scripts/build/packages.sh

@@ -0,0 +1,112 @@
+#! /bin/bash
+
+# -eE :: exit on error (do this in functions as well)
+# -u  :: show (and exit) when using unset variables
+# -o pipefail :: exit on error in pipes
+set -eE -u -o pipefail
+
+# shellcheck source=../helpers/log.sh
+source /usr/local/bin/helpers/log.sh
+
+_log_level_is 'trace' && QUIET='-y' || QUIET='-qq'
+
+function _pre_installation_steps
+{
+  _log 'info' 'Starting package installation'
+  _log 'debug' 'Running pre-installation steps'
+
+  _log 'trace' 'Updating package signatures'
+  apt-get "${QUIET}" update
+
+  _log 'trace' 'Installing packages that are needed early'
+  apt-get "${QUIET}" install --no-install-recommends apt-utils 2>/dev/null
+
+  _log 'trace' 'Upgrading packages'
+  apt-get "${QUIET}" upgrade
+}
+
+function _install_postfix
+{
+  _log 'debug' 'Installing Postfix'
+
+  _log 'warn' 'Applying workaround for Postfix bug (see https://github.com//issues/2023#issuecomment-855326403)'
+
+  # Debians postfix package has a post-install script that expects a valid FQDN hostname to work:
+  mv /bin/hostname /bin/hostname.bak
+  echo "echo 'docker-mailserver.invalid'" >/bin/hostname
+  chmod +x /bin/hostname
+  apt-get "${QUIET}" install --no-install-recommends postfix
+  mv /bin/hostname.bak /bin/hostname
+}
+
+function _install_packages
+{
+  _log 'debug' 'Installing all packages now'
+
+  declare -a DOVECOT_PACKAGES ANTI_VIRUS_SPAM_PACKAGES
+  declare -a CODECS_PACKAGES MISCELLANEOUS_PACKAGES
+  declare -a POSTFIX_PACKAGES MAIL_PROGRAMS_PACKAGES
+
+  DOVECOT_PACKAGES=(
+    dovecot-core dovecot-fts-xapian dovecot-imapd
+    dovecot-ldap dovecot-lmtpd dovecot-managesieved
+    dovecot-pop3d dovecot-sieve dovecot-solr
+  )
+
+  ANTI_VIRUS_SPAM_PACKAGES=(
+    amavisd-new clamav clamav-daemon
+    fail2ban pyzor razor spamassassin
+  )
+
+  CODECS_PACKAGES=(
+    altermime arj bzip2
+    cabextract cpio file
+    gzip lhasa liblz4-tool
+    lrzip lzop nomarch
+    p7zip-full pax rpm2cpio
+    unrar-free unzip xz-utils
+  )
+
+  MISCELLANEOUS_PACKAGES=(
+    apt-transport-https binutils bsd-mailx
+    ca-certificates curl dbconfig-no-thanks
+    dumb-init ed gamin gnupg iproute2
+    libdate-manip-perl libldap-common
+    libmail-spf-perl libnet-dns-perl
+    locales logwatch netcat-openbsd
+    nftables rsyslog supervisor
+    uuid whois
+  )
+
+  POSTFIX_PACKAGES=(
+    pflogsumm postgrey postfix-ldap
+    postfix-pcre postfix-policyd-spf-python postsrsd
+  )
+
+  MAIL_PROGRAMS_PACKAGES=(
+    fetchmail opendkim opendkim-tools
+    opendmarc libsasl2-modules sasl2-bin
+  )
+
+  apt-get "${QUIET}" --no-install-recommends install \
+    "${DOVECOT_PACKAGES[@]}" \
+    "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \
+    "${CODECS_PACKAGES[@]}" \
+    "${MISCELLANEOUS_PACKAGES[@]}" \
+    "${POSTFIX_PACKAGES[@]}" \
+    "${MAIL_PROGRAMS_PACKAGES[@]}"
+}
+
+function _post_installation_steps
+{
+  _log 'debug' 'Running post-installation steps (cleanup)'
+  apt-get "${QUIET}" clean
+  rm -rf /var/lib/apt/lists/*
+
+  _log 'info' 'Finished installing packages'
+}
+
+_pre_installation_steps
+_install_postfix
+_install_packages
+_post_installation_steps

test/linting/.hadolint.yaml

@@ -1,9 +1,4 @@
-ignored:
+ignored: [DL3008]
-  - DL3005
-  - DL3008
-  - DL3015
-  - DL3005
-
 trustedRegistries:
   - docker.io
   - quay.io