Clarify and enforce container image reproducibility #38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Docker reproducibility" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "*.*.*" | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| prepare-context: | |
| name: Prepare deterministic build context | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository with tags | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Create deterministic build context | |
| shell: bash | |
| run: ./scripts/reproduce-image.sh context "${PWD}/docker-context.tar" | |
| - name: Upload deterministic build context | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docker-build-context | |
| path: | | |
| docker-context.tar | |
| docker-context.tar.sha256 | |
| build-metadata.json | |
| if-no-files-found: error | |
| retention-days: 1 | |
| build-image: | |
| name: Build image (${{ matrix.build }}) | |
| needs: prepare-context | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # The second build deliberately differs in every way that must not | |
| # affect the image, so that sensitivity to the build environment is | |
| # caught here rather than years later when an image has to be rebuilt. | |
| include: | |
| - build: one | |
| context_subdir: ctx | |
| timezone: UTC | |
| locale: C.UTF-8 | |
| umask: "022" | |
| - build: two | |
| context_subdir: nested/much/deeper/workspace/ctx | |
| timezone: Pacific/Kiritimati | |
| locale: C | |
| umask: "077" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Download deterministic build context | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: docker-build-context | |
| path: repro-input | |
| # Both builds in this job run on the same runner image, so a builder | |
| # upgrade affects them equally and they still agree with each other. | |
| # Reporting it as a warning keeps that visible without blocking unrelated | |
| # work; the scheduled historical rebuild is where a builder change is | |
| # compared against digests recorded years earlier. | |
| - name: Check builder toolchain | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if ! output=$(./scripts/reproduce-image.sh toolchain "${PWD}" 2>&1); then | |
| echo "${output}" | |
| exit 1 | |
| fi | |
| echo "${output}" | |
| if printf '%s' "${output}" | grep -q "newer than the highest verified"; then | |
| echo "::warning::The build ran on a builder newer than the highest verified version. Confirm the layers are unchanged and then raise the values in docker/toolchain.env." | |
| fi | |
| - name: Build image | |
| shell: bash | |
| env: | |
| TZ: ${{ matrix.timezone }} | |
| LC_ALL: ${{ matrix.locale }} | |
| run: | | |
| set -euo pipefail | |
| umask ${{ matrix.umask }} | |
| (cd repro-input && sha256sum --check docker-context.tar.sha256) | |
| SOURCE_COMMIT=$(./scripts/reproduce-image.sh metadata repro-input/build-metadata.json source_commit) | |
| SOURCE_DATE_EPOCH=$(./scripts/reproduce-image.sh metadata repro-input/build-metadata.json source_date_epoch) | |
| SCITT_VERSION_OVERRIDE=$(./scripts/reproduce-image.sh metadata repro-input/build-metadata.json scitt_version) | |
| CONTEXT_SHA256=$(./scripts/reproduce-image.sh metadata repro-input/build-metadata.json context_sha256) | |
| export SOURCE_COMMIT SOURCE_DATE_EPOCH SCITT_VERSION_OVERRIDE CONTEXT_SHA256 | |
| tag="scitt-reproducibility:${{ matrix.build }}" | |
| context="${RUNNER_TEMP}/${{ matrix.context_subdir }}" | |
| ./scripts/reproduce-image.sh extract repro-input/docker-context.tar "${context}" | |
| ./scripts/reproduce-image.sh build "${context}" "${tag}" | |
| ./scripts/reproduce-image.sh manifest "${tag}" repro-input/reproduce.json "${context}" | |
| printf '%s' "$(docker image inspect --format '{{.Id}}' "${tag}")" > repro-input/image-id.txt | |
| docker image inspect \ | |
| --format '{{range .RootFS.Layers}}{{println .}}{{end}}' \ | |
| "${tag}" > repro-input/layer-digests.txt | |
| docker run \ | |
| --rm \ | |
| --entrypoint /bin/bash \ | |
| "${tag}" \ | |
| -c ' | |
| set -euo pipefail | |
| find / -xdev \ | |
| ! -path /etc/hostname \ | |
| ! -path /etc/hosts \ | |
| ! -path /etc/resolv.conf \ | |
| -printf "%y|%m|%U|%G|%T@|%s|%p|%l\n" | | |
| LC_ALL=C sort | |
| find / -xdev -type f \ | |
| ! -path /etc/hostname \ | |
| ! -path /etc/hosts \ | |
| ! -path /etc/resolv.conf \ | |
| -exec sha256sum {} + | | |
| LC_ALL=C sort -k2 | |
| ' > repro-input/filesystem-manifest.txt | |
| docker run \ | |
| --rm \ | |
| --entrypoint sha256sum \ | |
| "${tag}" \ | |
| /var/lib/rpm/rpmdb.sqlite > repro-input/rpmdb-sha256.txt | |
| - name: Upload image ID | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docker-image-id-${{ matrix.build }} | |
| path: | | |
| repro-input/image-id.txt | |
| repro-input/layer-digests.txt | |
| repro-input/filesystem-manifest.txt | |
| repro-input/rpmdb-sha256.txt | |
| repro-input/reproduce.json | |
| if-no-files-found: error | |
| retention-days: 1 | |
| compare-images: | |
| name: Compare filesystem layers | |
| needs: build-image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download image IDs | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: docker-image-id-* | |
| path: image-ids | |
| - name: Compare filesystem layers | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| first_id=image-ids/docker-image-id-one/image-id.txt | |
| second_id=image-ids/docker-image-id-two/image-id.txt | |
| first_layers=image-ids/docker-image-id-one/layer-digests.txt | |
| second_layers=image-ids/docker-image-id-two/layer-digests.txt | |
| first_rpmdb=image-ids/docker-image-id-one/rpmdb-sha256.txt | |
| second_rpmdb=image-ids/docker-image-id-two/rpmdb-sha256.txt | |
| echo "Build one image ID: $(cat "${first_id}")" | |
| echo "Build two image ID: $(cat "${second_id}")" | |
| if ! cmp --silent "${first_layers}" "${second_layers}"; then | |
| echo "Filesystem layer digest differences:" | |
| diff -u "${first_layers}" "${second_layers}" || true | |
| echo "Build one RPM database: $(cat "${first_rpmdb}")" | |
| echo "Build two RPM database: $(cat "${second_rpmdb}")" | |
| if ! cmp --silent "${first_rpmdb}" "${second_rpmdb}"; then | |
| echo "::error::RPM database content differs: build one $(cut -d ' ' -f 1 "${first_rpmdb}"), build two $(cut -d ' ' -f 1 "${second_rpmdb}")" | |
| fi | |
| echo "Filesystem differences:" | |
| filesystem_diff=$(diff -u \ | |
| image-ids/docker-image-id-one/filesystem-manifest.txt \ | |
| image-ids/docker-image-id-two/filesystem-manifest.txt || true) | |
| substantive_diff=$(printf '%s\n' "${filesystem_diff}" | | |
| grep -Ev '^([-+]{3}|@@|[-+]d\|555\|0\|0\|[^|]+\|0\|/(proc|sys)\|)' || true) | |
| if [ -n "${substantive_diff}" ]; then | |
| echo "${substantive_diff}" | |
| while IFS= read -r line; do | |
| echo "::error::Filesystem difference: ${line}" | |
| done < <(printf '%s\n' "${substantive_diff}" | head -n 30) | |
| else | |
| echo "::error::Image filesystem manifests match after excluding runtime /proc and /sys mountpoint timestamps; the layer archive differs in metadata not captured by the manifest, such as extended attributes or entry encoding." | |
| fi | |
| echo "::error::The Docker image filesystem layers are not reproducible. Build one: $(paste -sd, "${first_layers}"); build two: $(paste -sd, "${second_layers}")" | |
| exit 1 | |
| fi | |
| if ! cmp --silent "${first_id}" "${second_id}"; then | |
| echo "::warning::Filesystem layers match, but image configuration IDs differ. Pipeline labels and builder metadata are outside this gate's scope." | |
| fi | |
| echo "Docker image filesystem layers are reproducible:" | |
| cat "${first_layers}" | |
| - name: Upload reproduction manifest | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: docker-reproduce-manifest | |
| path: image-ids/docker-image-id-one/reproduce.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| publish-manifest: | |
| name: Publish reproduction manifest | |
| needs: compare-images | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| # Workflow artifacts expire, but reproducing a release years from now | |
| # requires the exact inputs this build used. Attach them to the release | |
| # so the recorded layer digests outlive artifact retention and can be | |
| # checked by the scheduled re-verification and by third parties. | |
| - name: Download verified build outputs | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: docker-image-id-one | |
| path: manifest | |
| - name: Collect release assets | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p release-assets | |
| cp manifest/reproduce.json release-assets/reproduce.json | |
| cp manifest/layer-digests.txt release-assets/image-layers.txt | |
| echo "Publishing reproduction manifest:" | |
| cat release-assets/reproduce.json | |
| - name: Attach manifest to the release | |
| uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 | |
| with: | |
| files: | | |
| release-assets/reproduce.json | |
| release-assets/image-layers.txt |