diff --git a/.github/actions/setup-linux-dependencies/action.yml b/.github/actions/setup-linux-dependencies/action.yml index 4f33a8322c..7f2325223d 100644 --- a/.github/actions/setup-linux-dependencies/action.yml +++ b/.github/actions/setup-linux-dependencies/action.yml @@ -1,129 +1,20 @@ name: Setup Linux dependencies -description: Install and cache the system packages required to build ReScript. +description: Install the system packages required to build ReScript. runs: using: composite steps: - # This is a small, repository-specific version of the filesystem cache used - # by awalsh128/cache-apt-pkgs-action@v1.4.3. Restoring the installed files is - # considerably faster than downloading and installing the .deb files again. - - name: Resolve package versions - id: packages + # Installed with apt, so that dpkg knows them: opam checks system + # packages with dpkg, and an earlier cache that unpacked these files + # without dpkg made opam take them for missing. musl-tools is needed for + # the static builds; bubblewrap and rsync for opam builds in a restored + # environment, such as coverage's bisect_ppx pin; cmake for OPAM. + # ocaml/setup-ocaml installs its own packages, such as the multilib ones, + # when it builds an environment. + - name: Install system dependencies shell: bash - env: - # Dependencies required by setup-ocaml, plus cmake for OPAM. The - # multilib packages are only available on x64 runners. - PACKAGES: >- - bubblewrap musl-tools rsync cmake - ${{ runner.arch == 'X64' && 'g++-multilib gcc-multilib' || '' }} - CACHE_VERSION: v5 run: | - set -euo pipefail - - read -ra packages <<< "$PACKAGES" - mapfile -t packages < <(printf '%s\n' "${packages[@]}" | sort -u) - - candidate() { - local output - output="$(apt-cache policy "$1" 2>&1)" || { - echo "apt-cache policy $1 failed with exit code $?:" >&2 - printf '%s\n' "$output" >&2 - return 1 - } - awk '/Candidate:/ { print $2; exit }' <<< "$output" - } - - # A runner image may ship without package lists, in which case no - # candidate resolves until the lists are fetched. - updated=false - resolved_packages=() - for package in "${packages[@]}"; do - version="$(candidate "$package")" || version="" - if [[ ( -z "$version" || "$version" == "(none)" ) && "$updated" == false ]]; then - sudo apt-get update - updated=true - version="$(candidate "$package")" || version="" - fi - if [[ -z "$version" || "$version" == "(none)" ]]; then - echo "Unable to resolve an APT candidate for $package" >&2 - exit 1 - fi - resolved_packages+=("$package=$version") - done - - cache_key="$({ - printf '%s\n' "$CACHE_VERSION" - (. /etc/os-release; printf '%s\n' "$ID" "$VERSION_ID") - dpkg --print-architecture - printf '%s\n' "${resolved_packages[@]}" - } | sha256sum | cut -d' ' -f1)" - - cache_dir="$RUNNER_TEMP/cache-apt-pkgs" - mkdir -p "$cache_dir" - printf '%s\n' "${resolved_packages[@]}" > "$cache_dir/requested-packages.txt" - echo "cache-dir=$cache_dir" >> "$GITHUB_OUTPUT" - echo "cache-key=$cache_key" >> "$GITHUB_OUTPUT" - - - name: Restore system dependencies - id: cache - uses: actions/cache/restore@v6 - with: - path: ${{ steps.packages.outputs.cache-dir }} - key: cache-apt-pkgs-${{ runner.os }}-${{ steps.packages.outputs.cache-key }} - - - name: Restore cached system dependencies - if: steps.cache.outputs.cache-hit == 'true' - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: sudo tar -xf "$CACHE_DIR/packages.tar" -C / - - - name: Install and cache system dependencies - if: steps.cache.outputs.cache-hit != 'true' - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: | - set -euo pipefail - - sudo apt-get update - mapfile -t packages < <(cut -d= -f1 "$CACHE_DIR/requested-packages.txt") - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + sudo apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes -qq \ --no-install-recommends --no-upgrade \ - "${packages[@]}" \ - | tee "$CACHE_DIR/install.log" - - sed -nE 's/^Unpacking ([^ :]+)(:[^ ]+)? .*\(([^ )]+).*/\1\2/p' \ - "$CACHE_DIR/install.log" \ - | sort -u \ - > "$CACHE_DIR/installed-packages.txt" - - if [[ -s "$CACHE_DIR/installed-packages.txt" ]]; then - while IFS= read -r package; do - dpkg-query --listfiles "$package" - done < "$CACHE_DIR/installed-packages.txt" \ - | sort -u \ - | while IFS= read -r path; do - if [[ -f "$path" || -L "$path" ]]; then - printf '%s\n' "${path#/}" - fi - done \ - | sudo tar -cf "$CACHE_DIR/packages.tar" -C / -T - - else - # A newer runner image may already contain every requested package. - sudo tar -cf "$CACHE_DIR/packages.tar" -T /dev/null - fi - - - name: Save system dependencies - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v6 - with: - path: ${{ steps.packages.outputs.cache-dir }} - key: ${{ steps.cache.outputs.cache-primary-key }} - - - name: Clean up system dependency cache - if: always() - shell: bash - env: - CACHE_DIR: ${{ steps.packages.outputs.cache-dir }} - run: rm -rf "$CACHE_DIR" + bubblewrap musl-tools rsync cmake diff --git a/.github/actions/setup-ocaml/action.yml b/.github/actions/setup-ocaml/action.yml index 7d8fc79995..d7c42129c7 100644 --- a/.github/actions/setup-ocaml/action.yml +++ b/.github/actions/setup-ocaml/action.yml @@ -1,6 +1,13 @@ name: Setup OCaml description: Restore or install the repository OPAM environment. +# An environment is only ever used as it was built: restored on an exact key, +# or built from scratch, never restored from other dependencies and updated. +# Pushes save what they build. A pull request that changes the opam files +# saves its environment too, in its own cache scope, so that only its first +# push per change builds from scratch. Each save deletes the earlier +# generations of the same key prefix on the same ref. + inputs: compiler: description: OCaml compiler switch specification. @@ -81,8 +88,9 @@ runs: dependency_flags_key="${DEPENDENCY_FLAGS//--/}" dependency_flags_key="${dependency_flags_key// /_}" dependency_hash="${{ hashFiles('*.opam') }}" - key="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-$dependency_hash" - echo "value=${key//,/-}" >> "$GITHUB_OUTPUT" + prefix="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-" + echo "prefix=${prefix//,/-}" >> "$GITHUB_OUTPUT" + echo "value=${prefix//,/-}$dependency_hash" >> "$GITHUB_OUTPUT" echo "setup-version=3.9.0" >> "$GITHUB_OUTPUT" - name: Restore OPAM environment @@ -140,10 +148,29 @@ runs: fi opam install . "${dependency_flags[@]}" + # A pull request that changes the dependencies saves its environment, so + # that its next pushes find it. Caches saved by pull_request runs are + # scoped to the pull request's merge ref, so no other branch sees them. + # Other pull requests don't save: each would store a copy of master's. + - name: Check whether the pull request changes the dependencies + id: dependencies + if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'pull_request' + shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + if git fetch --quiet --depth=1 origin "$BASE_SHA" && + ! git diff --quiet FETCH_HEAD HEAD -- '*.opam'; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + - name: Cache OPAM environment - # Caches created by pull_request runs are scoped to that PR's merge ref - # and cannot seed other PRs or branches, so only push runs save them. - if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'push' + id: save + if: >- + steps.cache.outputs.cache-hit != 'true' && + (github.event_name == 'push' || steps.dependencies.outputs.changed == 'true') uses: actions/cache/save@v6 with: path: | @@ -155,6 +182,26 @@ runs: C:\.opam key: ${{ steps.cache-key.outputs.value }} + # Keeps one environment per key prefix and ref. Needs actions: write; + # without it, the earlier generations stay until GitHub evicts them. + - name: Delete earlier OPAM environments + if: steps.save.outcome == 'success' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PREFIX: ${{ steps.cache-key.outputs.prefix }} + KEEP: ${{ steps.cache-key.outputs.value }} + run: | + gh api --paginate --method GET "repos/$GITHUB_REPOSITORY/actions/caches" \ + -f key="$PREFIX" -f ref="$GITHUB_REF" \ + --jq ".actions_caches[] | select(.key != \"$KEEP\") | .id" | + while IFS= read -r id; do + echo "Deleting cache $id" + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" || + echo "::warning::Couldn't delete cache $id." + done || + echo "::warning::Couldn't list the earlier OPAM environments." + - name: Use cached OPAM environment if: steps.cache.outputs.cache-hit == 'true' shell: bash diff --git a/.github/workflows/cache_cleanup.yml b/.github/workflows/cache_cleanup.yml new file mode 100644 index 0000000000..faeb849182 --- /dev/null +++ b/.github/workflows/cache_cleanup.yml @@ -0,0 +1,36 @@ +name: Cache Cleanup + +# Deletes the caches of a closed pull request: they are scoped to its merge +# ref, so nothing can use them any more, but they count towards the +# repository's cache storage until GitHub evicts them. The job runs no code +# of the pull request. + +on: + pull_request_target: + types: [closed] + workflow_dispatch: + inputs: + pr-number: + description: The pull request whose caches to delete. + required: true + +jobs: + cleanup: + runs-on: ubuntu-26.04 + permissions: + actions: write + steps: + - name: Delete the pull request's caches + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr-number }} + run: | + [[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || { echo "Not a pull request number: $PR_NUMBER" >&2; exit 1; } + gh api --paginate --method GET "repos/$GITHUB_REPOSITORY/actions/caches" \ + -f ref="refs/pull/$PR_NUMBER/merge" \ + --jq '.actions_caches[] | "\(.id) \(.key)"' | + while read -r id key; do + echo "Deleting $key" + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches/$id" || + echo "::warning::Couldn't delete cache $id." + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abb17e3667..916c515a42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,18 +104,17 @@ jobs: id: compiler-build-state-key shell: bash run: | - echo "value=compiler-build-state-v1-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v2-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('*.opam') }}" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" + # Only _build: dune's shared cache under ~/.cache/dune would grow with + # every master push that restores, extends and saves it again. - name: Restore compiler build state if: github.base_ref == 'master' || github.ref == 'refs/heads/master' id: compiler-build-state uses: actions/cache/restore@v6 with: - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build compiler @@ -143,10 +142,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/master' uses: actions/cache/save@v6 with: - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Verify promoted compiler exes @@ -456,7 +452,7 @@ jobs: pull-requests: write env: # The same build as the first linux-x64 build-compiler job, so that the - # OPAM environment and the dune cache are shared. + # OPAM environment and the build state are shared. DUNE_PROFILE: static OCAML_COMPILER: ocaml-variants.5.5.2+options,ocaml-option-static steps: @@ -492,7 +488,7 @@ jobs: - name: Compiler build state key id: compiler-build-state-key run: | - echo "value=compiler-build-state-v1-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v2-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER-${{ hashFiles('*.opam') }}" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" - name: Restore compiler build state @@ -500,10 +496,7 @@ jobs: with: # The paths are part of the cache's version, so they must be the # ones build-compiler saves. - path: | - C:\.cache\dune - ~/.cache/dune - _build + path: _build key: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build the compiler diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 606a1b44c2..8541a1eb2c 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -17,7 +17,8 @@ jobs: coverage: runs-on: ubuntu-26.04-arm permissions: - actions: write # For build cache overwriting + contents: read + actions: write # For deleting earlier OPAM environments id-token: write # For CodeCov OIDC env: OS: ubuntu-26.04-arm @@ -47,48 +48,11 @@ jobs: git+https://github.com/patricoferris/bisect_ppx.git#7061d643ff492b0045796357ee6917ded21fb1f0 --yes - # --- Coverage build cache -------------------------------------------- - # The bisect_ppx-instrumented dune build is a separate artifact from - # the main CI build, so it gets its own key. Only restore/save on - # master pushes — PRs share the master cache to stay fast but won't - # poison it. - - name: Coverage build state key - id: coverage-build-state-key - run: echo "value=coverage-build-state-v1-${{ env.OS }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ env.OCAML_COMPILER }}-${{ hashFiles('*.opam', 'compiler/**', 'dune-project') }}" >> $GITHUB_OUTPUT - - - name: Restore coverage build state - if: github.base_ref == 'master' || github.ref == 'refs/heads/master' - id: coverage-build-state - uses: actions/cache/restore@v6 - with: - path: | - ~/.cache/dune - _build - key: ${{ steps.coverage-build-state-key.outputs.value }} - + # No build state is cached: every compiler change misses it, and the + # instrumented build takes about 30 s. - name: Run coverage run: opam exec -- make coverage - - name: Delete stale coverage build state - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - gh extension install actions/gh-actions-cache - gh actions-cache delete ${{ steps.coverage-build-state-key.outputs.value }} \ - -R ${{ github.repository }} \ - -B "$GITHUB_REF" \ - --confirm || echo "not exist" - env: - GH_TOKEN: ${{ github.token }} - - - name: Save coverage build state - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - uses: actions/cache/save@v6 - with: - path: | - ~/.cache/dune - _build - key: ${{ steps.coverage-build-state-key.outputs.value }} - - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: