ci: support manual image rebuilds #421
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: docker-images | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - docker-test | |
| pull_request: | |
| branches: | |
| - master | |
| - docker-test | |
| schedule: | |
| - cron: '17 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| images: | |
| description: Images to rebuild (comma-separated, or all) | |
| required: false | |
| default: all | |
| type: string | |
| publish: | |
| description: Publish the rebuilt image tags | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| images: ${{ steps.filter.outputs.images }} | |
| has_images: ${{ steps.filter.outputs.has_images }} | |
| steps: | |
| - | |
| name: Checkout history | |
| uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - | |
| name: Select changed images | |
| id: filter | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} | |
| HEAD_SHA: ${{ github.sha }} | |
| DISPATCH_IMAGES: ${{ inputs.images }} | |
| run: | | |
| set -euo pipefail | |
| all_images='["ftpsync","nix-channels","rubygems-mirror","rustup-mirror","tsumugu","tunasync-scripts"]' | |
| if [ "$EVENT_NAME" = schedule ]; then | |
| images="$all_images" | |
| elif [ "$EVENT_NAME" = workflow_dispatch ]; then | |
| requested="$(printf '%s' "${DISPATCH_IMAGES:-all}" | tr -d '[:space:]')" | |
| if [ "$requested" = all ]; then | |
| images="$all_images" | |
| else | |
| selected='' | |
| IFS=',' | |
| for image in $requested; do | |
| case "$image" in | |
| ftpsync|nix-channels|rubygems-mirror|rustup-mirror|tsumugu|tunasync-scripts) | |
| selected="$selected $image" | |
| ;; | |
| *) | |
| echo "Unknown image: $image" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| done | |
| unset IFS | |
| images="$(printf '%s\n' $selected | sort -u | jq -Rsc 'split("\\n") | map(select(length > 0))')" | |
| fi | |
| else | |
| if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then | |
| BASE_SHA="$(git rev-parse "$HEAD_SHA^1")" | |
| fi | |
| changed="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")" | |
| selected='' | |
| global_changed=false | |
| for path in $changed; do | |
| case "$path" in | |
| .github/workflows/docker-images.yml) | |
| global_changed=true | |
| ;; | |
| .dockerignore) | |
| global_changed=true | |
| ;; | |
| Dockerfile) | |
| selected="$selected tunasync-scripts" | |
| ;; | |
| dockerfiles/ftpsync/*) selected="$selected ftpsync" ;; | |
| dockerfiles/nix-channels/*) selected="$selected nix-channels" ;; | |
| dockerfiles/rubygems-mirror/*) selected="$selected rubygems-mirror" ;; | |
| dockerfiles/rustup-mirror/*) selected="$selected rustup-mirror" ;; | |
| dockerfiles/tsumugu/*) selected="$selected tsumugu" ;; | |
| esac | |
| done | |
| if [ "$global_changed" = true ]; then | |
| images="$all_images" | |
| else | |
| # De-duplicate the image names selected by the changed paths. | |
| images="$(printf '%s\n' $selected | sort -u | jq -Rsc 'split("\\n") | map(select(length > 0))')" | |
| fi | |
| fi | |
| if [ "$images" = '[]' ]; then | |
| # Keep the matrix syntactically non-empty; the job-level condition skips it. | |
| echo 'images=["__none__"]' >> "$GITHUB_OUTPUT" | |
| echo 'has_images=false' >> "$GITHUB_OUTPUT" | |
| echo 'No Dockerfile or build workflow changes; image jobs skipped.' >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "images=$images" >> "$GITHUB_OUTPUT" | |
| echo 'has_images=true' >> "$GITHUB_OUTPUT" | |
| echo "Selected images: $images" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| crate_versions: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| rustup_mirror: ${{ steps.versions.outputs.rustup_mirror }} | |
| tsumugu: ${{ steps.versions.outputs.tsumugu }} | |
| steps: | |
| - name: Resolve stable crate versions | |
| id: versions | |
| run: | | |
| set -euo pipefail | |
| resolve_version() { | |
| local crate="$1" | |
| local output="$2" | |
| local version | |
| version="$( | |
| curl --fail --silent --show-error \ | |
| --retry 5 --retry-all-errors \ | |
| --user-agent 'tuna-container-builder (https://github.com/tuna/tunasync-scripts)' \ | |
| "https://crates.io/api/v1/crates/$crate" \ | |
| | jq -er '.crate.max_stable_version // .crate.max_version' | |
| )" | |
| echo "$output=$version" >> "$GITHUB_OUTPUT" | |
| echo "- \`$crate@$version\`" >> "$GITHUB_STEP_SUMMARY" | |
| } | |
| resolve_version rustup-mirror rustup_mirror | |
| resolve_version tsumugu tsumugu | |
| multi: | |
| needs: | |
| - changes | |
| - crate_versions | |
| if: needs.changes.outputs.has_images == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: ${{ fromJSON(needs.changes.outputs.images) }} | |
| os: | |
| - ubuntu-latest | |
| - ubuntu-24.04-arm | |
| include: | |
| - image: tunasync-scripts | |
| dockerfile_dir: . | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - | |
| name: Determine dockerfile dir | |
| id: docker_file_dir | |
| env: | |
| DF: ${{ matrix.dockerfile_dir }} | |
| IMG: ${{ matrix.image }} | |
| run: | | |
| if [ -z "${DF}" ]; then | |
| DF="dockerfiles/${IMG}" | |
| fi | |
| echo "docker_file_dir=${DF}" >> $GITHUB_OUTPUT | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v7 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - | |
| name: Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v6 | |
| with: | |
| images: tunathu/${{ matrix.image }} | |
| labels: | | |
| org.opencontainers.image.title=${{ matrix.image }} | |
| org.opencontainers.image.description=Runtime image for the ${{ matrix.image }} mirror job | |
| org.opencontainers.image.vendor=Tsinghua University TUNA Association | |
| annotations: | | |
| org.opencontainers.image.title=${{ matrix.image }} | |
| org.opencontainers.image.description=Runtime image for the ${{ matrix.image }} mirror job | |
| org.opencontainers.image.vendor=Tsinghua University TUNA Association | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v4 | |
| if: github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish) | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - | |
| name: Build and push | |
| uses: docker/build-push-action@v7 | |
| id: docker_build | |
| with: | |
| context: . | |
| file: ${{ steps.docker_file_dir.outputs.docker_file_dir }}/Dockerfile | |
| build-args: | | |
| CRATE_VERSION=${{ matrix.image == 'rustup-mirror' && needs.crate_versions.outputs.rustup_mirror || matrix.image == 'tsumugu' && needs.crate_versions.outputs.tsumugu || '' }} | |
| push: ${{ github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish) }} | |
| tags: tunathu/${{ matrix.image }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| annotations: ${{ steps.meta.outputs.annotations }} | |
| cache-from: type=gha,scope=${{ matrix.image }}-${{ runner.arch }} | |
| cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ runner.arch }} | |
| pull: true | |
| outputs: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish)) && 'type=image,push-by-digest=true,name-canonical=true,push=true' || 'type=image' }} | |
| - | |
| name: Export digest | |
| run: | | |
| mkdir -p ${{ runner.temp }}/digests/${{ matrix.image }} | |
| digest="${{ steps.docker_build.outputs.digest }}" | |
| touch "${{ runner.temp }}/digests/${{ matrix.image }}/${digest#sha256:}" | |
| - | |
| name: Upload digest | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: digests-${{ matrix.image }}-${{ runner.arch }} | |
| path: ${{ runner.temp }}/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - changes | |
| - multi | |
| if: (github.event_name == 'push' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish)) && needs.changes.outputs.has_images == 'true' && needs.multi.result == 'success' | |
| steps: | |
| - | |
| name: Determine docker tag | |
| id: docker_tag | |
| env: | |
| TAG_NAME: ${{ github.ref }} | |
| run: | | |
| if [ "${TAG_NAME##*/}" = "master" ]; then | |
| tag=latest | |
| else | |
| tag=build-test | |
| fi | |
| echo "docker_tag=${tag}" >> $GITHUB_OUTPUT | |
| - | |
| name: Download digests | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: ${{ runner.temp }}/digests | |
| pattern: digests-* | |
| merge-multiple: true | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - | |
| name: Login to DockerHub | |
| uses: docker/login-action@v4 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - | |
| name: Create manifest list and push | |
| working-directory: ${{ runner.temp }}/digests | |
| run: | | |
| set -euo pipefail | |
| created="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" | |
| source_url="${{ github.server_url }}/${{ github.repository }}" | |
| version="${{ steps.docker_tag.outputs.docker_tag }}" | |
| for i in *; do | |
| sources=() | |
| while IFS= read -r digest; do | |
| sources+=("tunathu/$i@sha256:$digest") | |
| done < <(find "$i" -maxdepth 1 -type f -printf '%f\n' | sort) | |
| target="tunathu/$i:${{ steps.docker_tag.outputs.docker_tag }}" | |
| current_json="$(docker buildx imagetools inspect "$target" --raw 2>/dev/null || true)" | |
| new_json="$(find "$i" -maxdepth 1 -type f -printf '%f\n' | jq -Rsc 'split("\\n") | map(select(length > 0) | "sha256:" + .) | sort')" | |
| old_json="$(jq -c '[.manifests[]? | select(.platform.os != "unknown") | .digest] | sort' <<< "$current_json" 2>/dev/null || echo '[]')" | |
| if [ "$old_json" != '[]' ] && [ "$old_json" = "$new_json" ]; then | |
| echo "$target already references the same platform manifests; skipping manifest push." | |
| continue | |
| fi | |
| docker buildx imagetools create -t "$target" \ | |
| --annotation "index:org.opencontainers.image.created=$created" \ | |
| --annotation "index:org.opencontainers.image.description=Runtime image for the $i mirror job" \ | |
| --annotation "index:org.opencontainers.image.revision=${{ github.sha }}" \ | |
| --annotation "index:org.opencontainers.image.source=$source_url" \ | |
| --annotation "index:org.opencontainers.image.title=$i" \ | |
| --annotation "index:org.opencontainers.image.url=$source_url" \ | |
| --annotation "index:org.opencontainers.image.vendor=Tsinghua University TUNA Association" \ | |
| --annotation "index:org.opencontainers.image.version=$version" \ | |
| "${sources[@]}" | |
| done | |
| - name: Inspect images | |
| working-directory: ${{ runner.temp }}/digests | |
| run: | | |
| for i in *; do | |
| DOCKER_IMG="tunathu/$i:${{ steps.docker_tag.outputs.docker_tag }}" | |
| echo "## $DOCKER_IMG" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| docker buildx imagetools inspect "$DOCKER_IMG" | tee -a $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo '| Platform | Compressed size | Digest |' >> $GITHUB_STEP_SUMMARY | |
| echo '| --- | ---: | --- |' >> $GITHUB_STEP_SUMMARY | |
| index_json="$(docker buildx imagetools inspect "$DOCKER_IMG" --raw)" | |
| while IFS=$'\t' read -r platform digest; do | |
| manifest_json="$(docker buildx imagetools inspect "tunathu/$i@$digest" --raw)" | |
| size_bytes="$(jq '[.layers[]?.size] | add // 0' <<< "$manifest_json")" | |
| size="$(numfmt --to=iec-i --suffix=B --format='%.1f' "$size_bytes")" | |
| printf '| `%s` | %s | `%s` |\n' "$platform" "$size" "$digest" \ | |
| >> $GITHUB_STEP_SUMMARY | |
| done < <(jq -r '.manifests[] | select(.platform.os != "unknown") | [(.platform.os + "/" + .platform.architecture), .digest] | @tsv' <<< "$index_json") | |
| done |