From 81a2dedc00e82dde51493f586295e6df08f90f9f Mon Sep 17 00:00:00 2001 From: Christoph Knittel Date: Fri, 13 Nov 2026 11:47:36 +0000 Subject: [PATCH] Save OPAM environments of dependency changes and bound the CI caches Why: A pull request that changed the opam files rebuilt every OPAM environment on each push (531 s on Linux x64 to 953 s on Windows), since only pushes to master save caches. The caches exceeded GitHub's 10 GB, so live ones were evicted: the coverage build state, keyed on a hash of compiler/**, saved a new 266 MB entry on nearly every master push, earlier generations stayed until GitHub's 7-day eviction, and the compiler build states grew with dune's shared cache. When an OPAM environment misses, a pull request that changes `*.opam` saves the one it built, in the cache scope of its merge ref, so its later pushes restore it. Each save deletes the earlier environments with the same key prefix on the same ref, and a new workflow deletes a closed pull request's caches. The coverage build state is gone, the compiler build states keep only `_build`, and the Linux system packages are installed with apt instead of unpacked from a cached tarball that dpkg didn't know about, so opam took them for missing. Review: Why not restore the environment of earlier dependencies and update it? The result would depend on what that environment held: pins, repository state, installed packages, depext status, global options. Environments are still only restored on an exact key, so each is the one a fresh build of its opam files produced. Review: Why do only pull requests that change the opam files save? Any other pull request would store a copy of master's environments, 3.3 GB each. Review: Was it tested? On this pull request: a temporary opam change built and saved all six environments, the next push restored them in 9-39 s, a second change replaced them, and reverting it restored master's again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Kbf3gGPsF3fHZn9rBxA7Hm Signed-off-by: Christoph Knittel --- .../setup-linux-dependencies/action.yml | 133 ++---------------- .github/actions/setup-ocaml/action.yml | 57 +++++++- .github/workflows/cache_cleanup.yml | 36 +++++ .github/workflows/ci.yml | 23 ++- .github/workflows/coverage.yml | 44 +----- 5 files changed, 112 insertions(+), 181 deletions(-) create mode 100644 .github/workflows/cache_cleanup.yml diff --git a/.github/actions/setup-linux-dependencies/action.yml b/.github/actions/setup-linux-dependencies/action.yml index 4f33a8322c..7f2325223d 100644 --- a/.github/actions/setup-linux-dependencies/action.yml +++ b/.github/actions/setup-linux-dependencies/action.yml @@ -1,129 +1,20 @@ name: Setup Linux dependencies -description: Install and cache the system packages required to build ReScript. +description: Install the system packages required to build ReScript. runs: using: composite steps: - # This is a small, repository-specific version of the filesystem cache used - # by awalsh128/cache-apt-pkgs-action@v1.4.3. Restoring the installed files is - # considerably faster than downloading and installing the .deb files again. - - name: Resolve package versions - id: packages + # Installed with apt, so that dpkg knows them: opam checks system + # packages with dpkg, and an earlier cache that unpacked these files + # without dpkg made opam take them for missing. musl-tools is needed for + # the static builds; bubblewrap and rsync for opam builds in a restored + # environment, such as coverage's bisect_ppx pin; cmake for OPAM. + # ocaml/setup-ocaml installs its own packages, such as the multilib ones, + # when it builds an environment. + - name: Install system dependencies shell: bash - env: - # Dependencies required by setup-ocaml, plus cmake for OPAM. The - # multilib packages are only available on x64 runners. - PACKAGES: >- - bubblewrap musl-tools rsync cmake - ${{ runner.arch == 'X64' && 'g++-multilib gcc-multilib' || '' }} - CACHE_VERSION: v5 run: | - set -euo pipefail - - read -ra packages <<< "$PACKAGES" - mapfile -t packages < <(printf '%s\n' "${packages[@]}" | sort -u) - - candidate() { - local output - output="$(apt-cache policy "$1" 2>&1)" || { - echo "apt-cache policy $1 failed with exit code $?:" >&2 - printf '%s\n' "$output" >&2 - return 1 - } - awk '/Candidate:/ { print $2; exit }' <<< "$output" - } - - # A runner image may ship without package lists, in which case no - # candidate resolves until the lists are fetched. - updated=false - resolved_packages=() - for package in "${packages[@]}"; do - version="$(candidate "$package")" || version="" - if [[ ( -z "$version" || "$version" == "(none)" ) && "$updated" == false ]]; then - sudo apt-get update - updated=true - version="$(candidate "$package")" || version="" - fi - if [[ -z "$version" || "$version" == "(none)" ]]; then - echo "Unable to resolve an APT candidate for $package" >&2 - exit 1 - fi - resolved_packages+=("$package=$version") - done - - cache_key="$({ - printf '%s\n' "$CACHE_VERSION" - (. /etc/os-release; printf '%s\n' "$ID" "$VERSION_ID") - dpkg --print-architecture - printf '%s\n' "${resolved_packages[@]}" - } | sha256sum | cut -d' ' -f1)" - - cache_dir="$RUNNER_TEMP/cache-apt-pkgs" - mkdir -p "$cache_dir" - printf '%s\n' "${resolved_packages[@]}" > "$cache_dir/requested-packages.txt" - echo "cache-dir=$cache_dir" >> "$GITHUB_OUTPUT" - echo "cache-key=$cache_key" >> "$GITHUB_OUTPUT" - - - name: Restore system dependencies - id: cache - uses: actions/cache/restore@v6 - with: - path: ${{ steps.packages.outputs.cache-dir }} - key: cache-apt-pkgs-${{ runner.os }}-${{ steps.packages.outputs.cache-key }} - - - name: Restore cached system dependencies - if: steps.cache.outputs.cache-hit == 'true' - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: sudo tar -xf "$CACHE_DIR/packages.tar" -C / - - - name: Install and cache system dependencies - if: steps.cache.outputs.cache-hit != 'true' - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: | - set -euo pipefail - - sudo apt-get update - mapfile -t packages < <(cut -d= -f1 "$CACHE_DIR/requested-packages.txt") - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes -qq \ --no-install-recommends --no-upgrade \ - "${packages[@]}" \ - | tee "$CACHE_DIR/install.log" - - sed -nE 's/^Unpacking ([^ :]+)(:[^ ]+)? .*\(([^ )]+).*/\1\2/p' \ - "$CACHE_DIR/install.log" \ - | sort -u \ - > "$CACHE_DIR/installed-packages.txt" - - if [[ -s "$CACHE_DIR/installed-packages.txt" ]]; then - while IFS= read -r package; do - dpkg-query --listfiles "$package" - done < "$CACHE_DIR/installed-packages.txt" \ - | sort -u \ - | while IFS= read -r path; do - if [[ -f "$path" || -L "$path" ]]; then - printf '%s\n' "${path#/}" - fi - done \ - | sudo tar -cf "$CACHE_DIR/packages.tar" -C / -T - - else - # A newer runner image may already contain every requested package. - sudo tar -cf "$CACHE_DIR/packages.tar" -T /dev/null - fi - - - name: Save system dependencies - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: ${{ steps.packages.outputs.cache-dir }} - key: ${{ steps.cache.outputs.cache-primary-key }} - - - name: Clean up system dependency cache - if: always() - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: rm -rf "$CACHE_DIR" + bubblewrap musl-tools rsync cmake diff --git a/.github/actions/setup-ocaml/action.yml b/.github/actions/setup-ocaml/action.yml index 7d8fc79995..d7c42129c7 100644 --- a/.github/actions/setup-ocaml/action.yml +++ b/.github/actions/setup-ocaml/action.yml @@ -1,6 +1,13 @@ name: Setup OCaml description: Restore or install the repository OPAM environment. +# An environment is only ever used as it was built: restored on an exact key, +# or built from scratch, never restored from other dependencies and updated. +# Pushes save what they build. A pull request that changes the opam files +# saves its environment too, in its own cache scope, so that only its first +# push per change builds from scratch. Each save deletes the earlier +# generations of the same key prefix on the same ref. + inputs: compiler: description: OCaml compiler switch specification. @@ -81,8 +88,9 @@ runs: dependency_flags_key="${DEPENDENCY_FLAGS//--/}" dependency_flags_key="${dependency_flags_key// /_}" dependency_hash="${{ hashFiles('*.opam') }}" - key="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-$dependency_hash" - echo "value=${key//,/-}" >> "$GITHUB_OUTPUT" + prefix="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-" + echo "prefix=${prefix//,/-}" >> "$GITHUB_OUTPUT" + echo "value=${prefix//,/-}$dependency_hash" >> "$GITHUB_OUTPUT" echo "setup-version=3.9.0" >> "$GITHUB_OUTPUT" - name: Restore OPAM environment @@ -140,10 +148,29 @@ runs: fi opam install . "${dependency_flags[@]}" + # A pull request that changes the dependencies saves its environment, so + # that its next pushes find it. Caches saved by pull_request runs are + # scoped to the pull request's merge ref, so no other branch sees them. + # Other pull requests don't save: each would store a copy of master's. + - name: Check whether the pull request changes the dependencies + id: dependencies + if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'pull_request' + shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if git fetch --quiet --depth=1 origin "$BASE_SHA" && + ! git diff --quiet FETCH_HEAD HEAD -- '*.opam'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + - name: Cache OPAM environment - # Caches created by pull_request runs are scoped to that PR's merge ref - # and cannot seed other PRs or branches, so only push runs save them. - if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'push' + id: save + if: >- + steps.cache.outputs.cache-hit != 'true' && + (github.event_name == 'push' || steps.dependencies.outputs.changed == 'true') uses: actions/cache/save@v6 with: path: | @@ -155,6 +182,26 @@ runs: C:\.opam key: ${{ steps.cache-key.outputs.value }} + # Keeps one environment per key prefix and ref. Needs actions: write; + # without it, the earlier generations stay until GitHub evicts them. + - name: Delete earlier OPAM environments + if: steps.save.outcome == 'success' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PREFIX: ${{ steps.cache-key.outputs.prefix }} + KEEP: ${{ steps.cache-key.outputs.value }} + run: | + gh api --paginate --method GET "repos/$GITHUB_REPOSITORY/actions/caches" \ + -f key="$PREFIX" -f ref="$GITHUB_REF" \ + --jq ".actions_caches[] | select(.key != \"$KEEP\") | .id" | + while IFS= read -r id; do + echo "Deleting cache $id" + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" || + echo "::warning::Couldn't delete cache $id." + done || + echo "::warning::Couldn't list the earlier OPAM environments." + - name: Use cached OPAM environment if: steps.cache.outputs.cache-hit == 'true' shell: bash diff --git a/.github/workflows/cache_cleanup.yml b/.github/workflows/cache_cleanup.yml new file mode 100644 index 0000000000..faeb849182 --- /dev/null +++ b/.github/workflows/cache_cleanup.yml @@ -0,0 +1,36 @@ +name: Cache Cleanup + +# Deletes the caches of a closed pull request: they are scoped to its merge +# ref, so nothing can use them any more, but they count towards the +# repository's cache storage until GitHub evicts them. The job runs no code +# of the pull request. + +on: + pull_request_target: + types: [closed] + workflow_dispatch: + inputs: + pr-number: + description: The pull request whose caches to delete. + required: true + +jobs: + cleanup: + runs-on: ubuntu-26.04 + permissions: + actions: write + steps: + - name: Delete the pull request's caches + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr-number }} + run: | + [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || { echo "Not a pull request number: $PR_NUMBER" >&2; exit 1; } + gh api --paginate --method GET "repos/$GITHUB_REPOSITORY/actions/caches" \ + -f ref="refs/pull/$PR_NUMBER/merge" \ + --jq '.actions_caches[] | "\(.id) \(.key)"' | + while read -r id key; do + echo "Deleting $key" + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" || + echo "::warning::Couldn't delete cache $id." + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb17e3667..916c515a42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,18 +104,17 @@ jobs: id: compiler-build-state-key shell: bash run: | - echo "value=compiler-build-state-v1-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v2-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('*.opam') }}" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" + # Only _build: dune's shared cache under ~/.cache/dune would grow with + # every master push that restores, extends and saves it again. - name: Restore compiler build state if: github.base_ref == 'master' || github.ref == 'refs/heads/master' id: compiler-build-state uses: actions/cache/restore@v6 with: - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build compiler @@ -143,10 +142,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/master' uses: actions/cache/save@v6 with: - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Verify promoted compiler exes @@ -456,7 +452,7 @@ jobs: pull-requests: write env: # The same build as the first linux-x64 build-compiler job, so that the - # OPAM environment and the dune cache are shared. + # OPAM environment and the build state are shared. DUNE_PROFILE: static OCAML_COMPILER: ocaml-variants.5.5.2+options,ocaml-option-static steps: @@ -492,7 +488,7 @@ jobs: - name: Compiler build state key id: compiler-build-state-key run: | - echo "value=compiler-build-state-v1-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v2-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER-${{ hashFiles('*.opam') }}" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" - name: Restore compiler build state @@ -500,10 +496,7 @@ jobs: with: # The paths are part of the cache's version, so they must be the # ones build-compiler saves. - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build the compiler diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 606a1b44c2..8541a1eb2c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,8 @@ jobs: coverage: runs-on: ubuntu-26.04-arm permissions: - actions: write # For build cache overwriting + contents: read + actions: write # For deleting earlier OPAM environments id-token: write # For CodeCov OIDC env: OS: ubuntu-26.04-arm @@ -47,48 +48,11 @@ jobs: git+https://github.com/patricoferris/bisect_ppx.git#7061d643ff492b0045796357ee6917ded21fb1f0 --yes - # --- Coverage build cache -------------------------------------------- - # The bisect_ppx-instrumented dune build is a separate artifact from - # the main CI build, so it gets its own key. Only restore/save on - # master pushes — PRs share the master cache to stay fast but won't - # poison it. - - name: Coverage build state key - id: coverage-build-state-key - run: echo "value=coverage-build-state-v1-${{ env.OS }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ env.OCAML_COMPILER }}-${{ hashFiles('*.opam', 'compiler/**', 'dune-project') }}" >> $GITHUB_OUTPUT - - - name: Restore coverage build state - if: github.base_ref == 'master' || github.ref == 'refs/heads/master' - id: coverage-build-state - uses: actions/cache/restore@v6 - with: - path: | - ~/.cache/dune - _build - key: ${{ steps.coverage-build-state-key.outputs.value }} - + # No build state is cached: every compiler change misses it, and the + # instrumented build takes about 30 s. - name: Run coverage run: opam exec -- make coverage - - name: Delete stale coverage build state - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - gh extension install actions/gh-actions-cache - gh actions-cache delete ${{ steps.coverage-build-state-key.outputs.value }} \ - -R ${{ github.repository }} \ - -B "$GITHUB_REF" \ - --confirm || echo "not exist" - env: - GH_TOKEN: ${{ github.token }} - - - name: Save coverage build state - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - uses: actions/cache/save@v6 - with: - path: | - ~/.cache/dune - _build - key: ${{ steps.coverage-build-state-key.outputs.value }} - - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: