From 32c508aa2a4d8899a3c95235650956596d97049f Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 21 Sep 2022 09:31:28 +0200 Subject: [PATCH] ci: enhance build process (#2755) This new script is a clean way of handling the installation of packages. I think the huge `RUN` command in `Dockerfile` was hard to read and maintain. Using a script is a non-issue, as the image is rebuilt whenever the script is touched. Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- Dockerfile | 67 +++------------ Makefile | 5 +- target/scripts/build/packages.sh | 112 ++++++++++++++++++++++++++ test/linting/.hadolint.yaml | 7 +- 5 files changed, 129 insertions(+), 64 deletions(-) create mode 100644 target/scripts/build/packages.sh diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 945b51c9..9f77035e 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "::set-output name=version::$(/dev/null && \ - apt-get -qq dist-upgrade && \ - echo "applying workaround for ubuntu/postfix bug described in https://github.com/docker-mailserver/docker-mailserver/issues/2023#issuecomment-855326403" && \ - mv /bin/hostname{,.bak} && \ - echo "echo docker-mailserver.invalid" > /bin/hostname && \ - chmod +x /bin/hostname && \ - apt-get -qq install postfix && \ - mv /bin/hostname{.bak,} && \ - apt-get -qq --no-install-recommends install \ - # A - D - altermime amavisd-new apt-transport-https arj binutils bzip2 bsd-mailx \ - ca-certificates cabextract clamav clamav-daemon cpio curl \ - dbconfig-no-thanks dovecot-core dovecot-fts-xapian dovecot-imapd \ - dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d \ - dovecot-sieve dovecot-solr dumb-init \ - # E - O - ed fetchmail file gamin gnupg gzip iproute2 \ - locales logwatch lhasa libdate-manip-perl libldap-common liblz4-tool \ - libmail-spf-perl libnet-dns-perl libsasl2-modules lrzip lzop \ - netcat-openbsd nftables nomarch opendkim opendkim-tools opendmarc \ - # P - Z - pax pflogsumm postgrey p7zip-full postfix-ldap postfix-pcre \ - postfix-policyd-spf-python postsrsd pyzor \ - razor rpm2cpio rsyslog sasl2-bin spamassassin supervisor \ - unrar-free unzip uuid whois xz-utils && \ - # Fail2Ban - gpg --keyserver ${FAIL2BAN_GPG_PUBLIC_KEY_SERVER} \ - --recv-keys ${FAIL2BAN_GPG_PUBLIC_KEY_ID} 2>&1 && \ - curl -Lkso fail2ban.deb ${FAIL2BAN_DEB_URL} && \ - curl -Lkso fail2ban.deb.asc ${FAIL2BAN_DEB_ASC_URL} && \ - FINGERPRINT=$(LANG=C gpg --verify \ - fail2ban.deb.asc fail2ban.deb 2>&1 \ - | sed -n 's#Primary key fingerprint: \(.*\)#\1#p') && \ - if [[ -z ${FINGERPRINT} ]]; then \ - echo "ERROR: Invalid GPG signature!" >&2; exit 1; fi && \ - if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]]; then \ - echo "ERROR: Wrong GPG fingerprint!" >&2; exit 1; fi && \ - dpkg -i fail2ban.deb 2>&1 && \ - rm fail2ban.deb fail2ban.deb.asc && \ - # cleanup - apt-get -qq autoremove && \ - apt-get -qq autoclean && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* && \ - c_rehash 2>&1 - +COPY target/scripts/build/* /build/ COPY target/scripts/helpers/log.sh /usr/local/bin/helpers/log.sh -COPY target/bin/sedfile /usr/local/bin/sedfile +RUN /bin/bash /build/packages.sh +COPY target/bin/sedfile /usr/local/bin/sedfile RUN chmod +x /usr/local/bin/sedfile + # ----------------------------------------------- # --- ClamAV & FeshClam ------------------------- # ----------------------------------------------- @@ -81,12 +33,15 @@ RUN chmod +x /usr/local/bin/sedfile RUN \ echo '0 */6 * * * clamav /usr/bin/freshclam --quiet' >/etc/cron.d/clamav-freshclam && \ chmod 644 /etc/clamav/freshclam.conf && \ - freshclam && \ sedfile -i 's/Foreground false/Foreground true/g' /etc/clamav/clamd.conf && \ mkdir /var/run/clamav && \ chown -R clamav:root /var/run/clamav && \ rm -rf /var/log/clamav/ +# Copy over latest DB updates from official ClamAV image. Better than running `freshclam` (which requires extra RAM during build) +# hadolint ignore=DL3021 +COPY --link --from=docker.io/clamav/clamav:latest /var/lib/clamav /var/lib/clamav + # ----------------------------------------------- # --- Dovecot ----------------------------------- # ----------------------------------------------- diff --git a/Makefile b/Makefile index 475c954e..caca6445 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,10 @@ VCS_VERSION = $(shell cat VERSION) all: lint build backup generate-accounts tests clean build: - docker build -t $(NAME) . --build-arg VCS_VERSION=$(VCS_VERSION) --build-arg VCS_REVISION=$(VCS_REVISION) + @ DOCKER_BUILDKIT=1 docker build --tag $(NAME) \ + --build-arg VCS_VERSION=$(VCS_VERSION) \ + --build-arg VCS_REVISION=$(VCS_REVISION) \ + . backup: # if backup directory exist, clean hasn't been called, therefore diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh new file mode 100644 index 00000000..34d95857 --- /dev/null +++ b/target/scripts/build/packages.sh @@ -0,0 +1,112 @@ +#! /bin/bash + +# -eE :: exit on error (do this in functions as well) +# -u :: show (and exit) when using unset variables +# -o pipefail :: exit on error in pipes +set -eE -u -o pipefail + +# shellcheck source=../helpers/log.sh +source /usr/local/bin/helpers/log.sh + +_log_level_is 'trace' && QUIET='-y' || QUIET='-qq' + +function _pre_installation_steps +{ + _log 'info' 'Starting package installation' + _log 'debug' 'Running pre-installation steps' + + _log 'trace' 'Updating package signatures' + apt-get "${QUIET}" update + + _log 'trace' 'Installing packages that are needed early' + apt-get "${QUIET}" install --no-install-recommends apt-utils 2>/dev/null + + _log 'trace' 'Upgrading packages' + apt-get "${QUIET}" upgrade +} + +function _install_postfix +{ + _log 'debug' 'Installing Postfix' + + _log 'warn' 'Applying workaround for Postfix bug (see https://github.com//issues/2023#issuecomment-855326403)' + + # Debians postfix package has a post-install script that expects a valid FQDN hostname to work: + mv /bin/hostname /bin/hostname.bak + echo "echo 'docker-mailserver.invalid'" >/bin/hostname + chmod +x /bin/hostname + apt-get "${QUIET}" install --no-install-recommends postfix + mv /bin/hostname.bak /bin/hostname +} + +function _install_packages +{ + _log 'debug' 'Installing all packages now' + + declare -a DOVECOT_PACKAGES ANTI_VIRUS_SPAM_PACKAGES + declare -a CODECS_PACKAGES MISCELLANEOUS_PACKAGES + declare -a POSTFIX_PACKAGES MAIL_PROGRAMS_PACKAGES + + DOVECOT_PACKAGES=( + dovecot-core dovecot-fts-xapian dovecot-imapd + dovecot-ldap dovecot-lmtpd dovecot-managesieved + dovecot-pop3d dovecot-sieve dovecot-solr + ) + + ANTI_VIRUS_SPAM_PACKAGES=( + amavisd-new clamav clamav-daemon + fail2ban pyzor razor spamassassin + ) + + CODECS_PACKAGES=( + altermime arj bzip2 + cabextract cpio file + gzip lhasa liblz4-tool + lrzip lzop nomarch + p7zip-full pax rpm2cpio + unrar-free unzip xz-utils + ) + + MISCELLANEOUS_PACKAGES=( + apt-transport-https binutils bsd-mailx + ca-certificates curl dbconfig-no-thanks + dumb-init ed gamin gnupg iproute2 + libdate-manip-perl libldap-common + libmail-spf-perl libnet-dns-perl + locales logwatch netcat-openbsd + nftables rsyslog supervisor + uuid whois + ) + + POSTFIX_PACKAGES=( + pflogsumm postgrey postfix-ldap + postfix-pcre postfix-policyd-spf-python postsrsd + ) + + MAIL_PROGRAMS_PACKAGES=( + fetchmail opendkim opendkim-tools + opendmarc libsasl2-modules sasl2-bin + ) + + apt-get "${QUIET}" --no-install-recommends install \ + "${DOVECOT_PACKAGES[@]}" \ + "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \ + "${CODECS_PACKAGES[@]}" \ + "${MISCELLANEOUS_PACKAGES[@]}" \ + "${POSTFIX_PACKAGES[@]}" \ + "${MAIL_PROGRAMS_PACKAGES[@]}" +} + +function _post_installation_steps +{ + _log 'debug' 'Running post-installation steps (cleanup)' + apt-get "${QUIET}" clean + rm -rf /var/lib/apt/lists/* + + _log 'info' 'Finished installing packages' +} + +_pre_installation_steps +_install_postfix +_install_packages +_post_installation_steps diff --git a/test/linting/.hadolint.yaml b/test/linting/.hadolint.yaml index caa8f1cf..28b8973a 100644 --- a/test/linting/.hadolint.yaml +++ b/test/linting/.hadolint.yaml @@ -1,9 +1,4 @@ -ignored: - - DL3005 - - DL3008 - - DL3015 - - DL3005 - +ignored: [DL3008] trustedRegistries: - docker.io - quay.io