github-com-docker-mailserver/docker-mailserver
1
0
Fork 0
mirror of https://github.com/docker-mailserver/docker-mailserver.git synced 2024-01-19 02:48:50 +00:00
Code Issues Projects Releases Wiki Activity

ci: improve GitHub Action CI with re-usable workflows (#2753)

Browse source 
Mew re-usable workflows are introduced to handle building, testing and publishing the container
image in a uniform and easy way. Now, the `scheduled_builds`, `default_on_push`
and a part of the `test_merge_requests` workflow can use the same code
for building, testing and publishing the container images. This is DRY.

Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com>
This commit is contained in:
Georg Lauterbach 2022-09-09 11:12:17 +02:00 committed by GitHub
parent 8bc8fc873c
commit f8e1bb0f42
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 303 additions and 308 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

133
.github/workflows/default_on_push.yml vendored
View file

@ -1,4 +1,4 @@
name: "Build, Test & Deploy"
name: 'Build, Test & Deploy'
on:
workflow_dispatch:
@ -6,11 +6,11 @@ on:
branches:
- master
paths:
- 'target/**'
- '.dockerignore'
- '.gitmodules'
- 'Dockerfile'
- 'setup.sh'
- target/**
- .dockerignore
- .gitmodules
- Dockerfile
- setup.sh
tags:
- '*.*.*'
@ -19,110 +19,21 @@ permissions:
packages: write
jobs:
build-and-test-image:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
build-image:
name: 'Build AMD64 Image'
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_build.yml@master
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
id: buildx
run-tests:
name: 'Test AMD64 Image'
needs: build-image
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_test.yml@master
with:
cache-key: ${{ needs.build-image.outputs.build-cache-key }}
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Build image locally
uses: docker/build-push-action@v3.1.1
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
platforms: linux/amd64
load: true
tags: mailserver-testing:ci
cache-to: type=local,dest=/tmp/.buildx-cache
- name: Run test suite
run: >
NAME=mailserver-testing:ci
bash -c 'make generate-accounts tests'
env:
CI: true
build-multiarch-and-publish:
needs: build-and-test-image
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
- name: Prepare tags
id: prep
uses: docker/metadata-action@v4.0.1
with:
images: |
${{ secrets.DOCKER_REPOSITORY }}
${{ secrets.GHCR_REPOSITORY }}
tags: |
type=edge,branch=master
type=semver,pattern={{major}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}.{{minor}}.{{patch}}
flavor: |
latest=auto
- name: Set up QEMU
uses: docker/setup-qemu-action@v2.0.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
id: buildx
- name: Cache Docker layers
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to DockerHub
uses: docker/login-action@v2.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build images locally
uses: docker/build-push-action@v3.1.1
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
platforms: linux/amd64,linux/arm/v7,linux/arm64
push: true
tags: ${{ steps.prep.outputs.tags }}
cache-from: type=local,src=/tmp/.buildx-cache
publish-images:
name: 'Publish AMD64 and ARM64 Image'
needs: [build-image, run-tests]
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_publish.yml@master
with:
cache-key: ${{ needs.build-image.outputs.build-cache-key }}
secrets: inherit

108
.github/workflows/generic_build.yml vendored Normal file
View file

@ -0,0 +1,108 @@
name: 'Build the DMS Container Image'
on:
workflow_call:
inputs:
platforms:
required: false
type: string
default: linux/amd64
permissions:
contents: read
# `actions/cache` does not upload a new cache until completing a job successfully.
# To better cache image builds, tests are handled in a dependent job afterwards.
# This way failing tests will not prevent caching of an image. Useful when the build context
# is not changed by new commits.
jobs:
build-image:
name: 'Build'
runs-on: ubuntu-20.04
outputs:
build-cache-key: ${{ steps.derive-image-cache-key.outputs.digest }}
steps:
- name: 'Checkout'
uses: actions/checkout@v3
with:
submodules: recursive
# Can potentially be replaced by: `${{ hashFiles('target/**', 'Dockerfile', 'VERSION') }}`
# Must not be affected by file metadata changes and have a consistent sort order:
# https://docs.github.com/en/actions/learn-github-actions/expressions#hashfiles
# Keying by the relevant build context is more re-usable than a commit SHA.
- name: 'Derive Docker image cache key from content'
id: derive-image-cache-key
shell: bash
run: |
ADDITIONAL_FILES=(
'Dockerfile'
'VERSION'
)
# Recursively collect file paths from `target/` and pipe a list of
# checksums to be sorted (by hash value) and finally generate a checksum
# of that list, using `awk` to only return the hash value (digest):
IMAGE_CHECKSUM=$(\
find ./target -type f -exec sha256sum "${ADDITIONAL_FILES[@]}" {} + \
| sort \
| sha256sum \
| awk '{ print $1 }' \
)
echo "::set-output name=digest::${IMAGE_CHECKSUM}"
# Attempts to restore the build cache from a prior build run.
# If the exact key is not restored, then upon a successful job run
# the new cache is uploaded for this key containing the contents at `path`.
# Cache storage has a limit of 10GB, and uploads expire after 7 days.
# When full, the least accessed cache upload is evicted to free up storage.
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
- name: 'Handle Docker build layer cache'
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: cache-buildx-${{ steps.derive-image-cache-key.outputs.digest }}
# If no exact cache-hit for key found, lookup caches with a `cache-buildx-` key prefix:
# This is safe due to cache layer invalidation via the image build context.
restore-keys: |
cache-buildx-
- name: 'Set up QEMU'
uses: docker/setup-qemu-action@v2.0.0
with:
platforms: arm64,arm
- name: 'Set up Docker Buildx'
uses: docker/setup-buildx-action@v2.0.0
# NOTE: AMD64 can build within 2 minutes, ARM adds 13 minutes. 330MB each
# ARMv7 can build in parallel, adding no extra time (but does add 150MB cache size).
- name: 'Build images'
uses: docker/build-push-action@v3.1.1
with:
context: .
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
# Build at least the AMD64 image (which runs against the test suite).
platforms: ${{ inputs.platforms }}
# Paired with steps `actions/cache` and `Replace cache` (replace src with dest):
# NOTE: `mode=max` is only for `cache-to`, it configures exporting all image layers.
# https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#cache-from
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
# This job just builds the image and stores to cache, no other exporting required:
# https://github.com/docker/build-push-action/issues/546#issuecomment-1122631106
outputs: type=cacheonly
# WORKAROUND: The `cache-to: type=local` input for `build-push-action` persists old-unused cache.
# The workaround is to write the new build cache to a different location that replaces the
# original restored cache after build, reducing frequency of eviction due to cache storage limit (10GB).
# https://github.com/docker/build-push-action/blob/965c6a410d446a30e95d35052c67d6eded60dad6/docs/advanced/cache.md?plain=1#L193-L199
# NOTE: This does not affect `cache-hit == 'true'` (which skips upload on direct cache key hit)
- name: 'Replace cache'
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache

79
.github/workflows/generic_publish.yml vendored Normal file
View file

@ -0,0 +1,79 @@
name: 'Publish the DMS Container Image'
on:
workflow_call:
inputs:
cache-key:
required: true
type: string
permissions:
contents: read
packages: write
jobs:
publish-images:
name: 'Publish'
runs-on: ubuntu-20.04
steps:
- name: 'Checkout'
uses: actions/checkout@v3
with:
submodules: recursive
- name: 'Prepare tags'
id: prep
uses: docker/metadata-action@v4.0
with:
images: |
${{ secrets.DOCKER_REPOSITORY }}
${{ secrets.GHCR_REPOSITORY }}
tags: |
type=edge,branch=master
type=semver,pattern={{major}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}.{{minor}}.{{patch}}
- name: 'Set up QEMU'
uses: docker/setup-qemu-action@v2
with:
platforms: arm64,arm
- name: 'Set up Docker Buildx'
uses: docker/setup-buildx-action@v2
# Try get the cached build layers from a prior `generic_build.yml` job.
# NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`),
# only AMD64 image is expected to be cached, ARM images will build from scratch.
- name: 'Retrieve image build from build cache'
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: cache-buildx-${{ inputs.cache-key }}
restore-keys: |
cache-buildx-
- name: 'Login to DockerHub'
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: 'Login to GitHub Container Registry'
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: 'Build and publish images'
uses: docker/build-push-action@v3.1
with:
context: .
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
platforms: linux/amd64,linux/arm/v7,linux/arm64
push: true
tags: ${{ steps.prep.outputs.tags }}
cache-from: type=local,src=/tmp/.buildx-cache

54
.github/workflows/generic_test.yml vendored Normal file
View file

@ -0,0 +1,54 @@
name: 'Test the DMS Container Image'
on:
workflow_call:
inputs:
cache-key:
required: true
type: string
permissions:
contents: read
jobs:
run-tests:
name: 'Test'
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
# Required to retrieve bats (core + extras):
submodules: recursive
# Get the cached build layers from the build job:
# This should always be a cache-hit, thus `restore-keys` fallback is not used.
# No new cache uploads should ever happen for this job.
- name: 'Retrieve image built from build cache'
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: cache-buildx-${{ inputs.cache-key }}
# Importing from the cache should create the image within approx 30 seconds:
# Earlier `buildx` + `qemu` steps are not needed as no cache is exported,
# and only a single platform (AMD64) is loaded:
- name: 'Build AMD64 image from cache'
uses: docker/build-push-action@v3.1.1
with:
context: .
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
tags: mailserver-testing:ci
# Export the built image to the Docker host for use with BATS:
load: true
# Rebuilds the AMD64 image from the cache:
platforms: linux/amd64
cache-from: type=local,src=/tmp/.buildx-cache
- name: 'Run tests'
run: make generate-accounts tests
env:
CI: true
NAME: mailserver-testing:ci

66
.github/workflows/scheduled_builds.yml vendored
View file

@ -1,60 +1,24 @@
name: "Build Edge on Schedule"
name: 'Deploy :edge on Schedule'
on:
schedule:
- cron: "0 0 * * 5"
- cron: 0 0 * * 5
permissions:
contents: read
packages: write
jobs:
publish:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
submodules: recursive
build-images:
name: 'Build Images'
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_build.yml@master
with:
platforms: linux/amd64,linux/arm/v7,linux/arm64
- name: Prepare tags
id: prep
uses: docker/metadata-action@v4.0.1
with:
images: |
${{ secrets.DOCKER_REPOSITORY }}
${{ secrets.GHCR_REPOSITORY }}
tags: |
type=raw,value=edge
- name: Set up QEMU
uses: docker/setup-qemu-action@v2.0.0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
id: buildx
- name: Login to DockerHub
uses: docker/login-action@v2.0.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2.0.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build image locally
uses: docker/build-push-action@v3.1.1
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./Dockerfile
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
platforms: linux/amd64,linux/arm/v7,linux/arm64
push: true
tags: ${{ steps.prep.outputs.tags }}
publish-images:
name: 'Publish Images'
needs: build-images
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_publish.yml@master
with:
cache-key: ${{ needs.build-images.outputs.build-cache-key }}
secrets: inherit

171
.github/workflows/test_merge_requests.yml vendored
View file

@ -1,158 +1,37 @@
name: "Test Merge Requests"
name: 'Test Merge Requests'
on:
workflow_dispatch:
pull_request:
paths:
- 'target/**'
- 'test/**'
- '.dockerignore'
- '.gitmodules'
- 'Dockerfile'
- 'setup.sh'
- target/**
- test/**
- .dockerignore
- .gitmodules
- Dockerfile
- setup.sh
permissions:
contents: read
# `actions/cache` does not upload a new cache until completing a job successfully.
# To better cache image builds, tests are handled in a dependent job afterwards.
# This way failing tests will not prevent caching of an image. Useful when the build context
# is not changed by new commits.
jobs:
job-build-image:
runs-on: ubuntu-20.04
outputs:
image-build-key: ${{ steps.derive-image-cache-key.outputs.digest }}
steps:
- name: 'Checkout'
uses: actions/checkout@v3
with:
# Required for image to include `configomat.sh`:
submodules: recursive
build-image-amd64:
name: 'Build AMD64 Image'
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_build.yml@master
# Can potentially be replaced by: `${{ hashFiles('target/**', 'Dockerfile', 'VERSION') }}`
# Must not be affected by file metadata changes and have a consistent sort order:
# https://docs.github.com/en/actions/learn-github-actions/expressions#hashfiles
# Keying by the relevant build context is more re-usable than a commit SHA.
- name: 'Derive Docker image cache key from content'
id: derive-image-cache-key
shell: bash
run: |
ADDITIONAL_FILES=(
'Dockerfile'
'VERSION'
)
run-tests:
name: 'Test AMD64 Image'
needs: build-image-amd64
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_test.yml@master
with:
cache-key: ${{ needs.build-image-amd64.outputs.build-cache-key }}
# Recursively collect file paths from `target/` and pipe a list of
# checksums to be sorted (by hash value) and finally generate a checksum
# of that list, using `awk` to only return the hash value (digest):
IMAGE_CHECKSUM=$(\
find ./target -type f -exec sha256sum "${ADDITIONAL_FILES[@]}" {} + \
| sort \
| sha256sum \
| awk '{ print $1 }' \
)
echo "::set-output name=digest::${IMAGE_CHECKSUM}"
# Attempts to restore the build cache from a prior build run.
# If the exact key is not restored, then upon a successful job run
# the new cache is uploaded for this key containing the contents at `path`.
# Cache storage has a limit of 10GB, and uploads expire after 7 days.
# When full, the least accessed cache upload is evicted to free up storage.
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows
- name: 'Handle Docker build layer cache'
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: cache-buildx-${{ steps.derive-image-cache-key.outputs.digest }}
# If no exact cache-hit for key found, lookup caches with a `cache-buildx-` key prefix:
# This is safe due to cache layer invalidation via the image build context.
restore-keys: |
cache-buildx-
# Support ARM64 builds on AMD64 host:
- name: 'Set up QEMU'
uses: docker/setup-qemu-action@v2.0.0
with:
platforms: arm64
# Enables `buildx` support within `build-push-action`, improving cache and platform support:
- name: 'Set up Docker Buildx'
uses: docker/setup-buildx-action@v2.0.0
# NOTE: AMD64 can build within 2 minutes, ARM adds 13 minutes. 330MB each
# ARMv7 can build in parallel, adding no extra time (but does add 150MB cache size).
# Moving ARM build to a separate job would cut down time to start running tests.
- name: 'Build images'
uses: docker/build-push-action@v3.1.1
with:
context: .
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
tags: mailserver-testing:ci
# Build for AMD64 (runs against test suite) and ARM64 (only to verify building works):
platforms: linux/amd64,linux/arm64
# Paired with steps `actions/cache` and `Replace cache` (replace src with dest):
# NOTE: `mode=max` is only for `cache-to`, it configures exporting all image layers.
# https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#cache-from
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
# This job just builds the image and stores to cache, no other exporting required:
# https://github.com/docker/build-push-action/issues/546#issuecomment-1122631106
outputs: type=cacheonly
# WORKAROUND: The `cache-to: type=local` input for `build-push-action` persists old-unused cache.
# The workaround is to write the new build cache to a different location that replaces the
# original restored cache after build, reducing frequency of eviction due to cache storage limit (10GB).
# https://github.com/docker/build-push-action/blob/965c6a410d446a30e95d35052c67d6eded60dad6/docs/advanced/cache.md?plain=1#L193-L199
# NOTE: This does not affect `cache-hit == 'true'` (which skips upload on direct cache key hit)
- name: 'Replace cache'
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
job-run-tests:
name: 'Run Test Suite'
needs: job-build-image
runs-on: ubuntu-20.04
steps:
- name: 'Checkout'
uses: actions/checkout@v3
with:
# Required to retrieve bats (core + extras):
submodules: recursive
# Get the cached build layers from the build job:
# This should always be a cache-hit, no new uploads should happen when this job finishes:
- name: 'Retrieve image build from build cache'
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache
key: cache-buildx-${{ needs.job-build-image.outputs.image-build-key }}
restore-keys: |
cache-buildx-
# Importing from the cache should create the image within approx 30 seconds:
# buildx not needed as no exporting and only single AMD64 platform is loaded:
- name: 'Build AMD64 image from cache'
uses: docker/build-push-action@v3.1.1
with:
context: .
build-args: |
VCS_REF=${{ github.sha }}
VCS_VER=${{ github.ref }}
tags: mailserver-testing:ci
# Export the built image for the Docker host to use:
load: true
# Rebuilds the AMD64 image from the cache:
platforms: linux/amd64
cache-from: type=local,src=/tmp/.buildx-cache
- name: 'Run tests'
env:
CI: true
run: |
NAME=mailserver-testing:ci
make generate-accounts tests
job-build-arm:
name: 'Build ARM64 Image'
# Dependency ensures the cache-key is only created for AMD64 builds.
# ARM64 will not be able to use this cache, building from scratch each time.
# Expect about 13 minutes build time until adopting `type=gha` with scopes for cache.
needs: build-image-amd64
uses: docker-mailserver/docker-mailserver/.github/workflows/generic_build.yml@master
with:
platforms: linux/arm/v7,linux/arm64