diff --git a/.github/actions/setup-ocaml/action.yml b/.github/actions/setup-ocaml/action.yml index 7d8fc79995..11834590b4 100644 --- a/.github/actions/setup-ocaml/action.yml +++ b/.github/actions/setup-ocaml/action.yml @@ -15,6 +15,13 @@ inputs: description: Flags passed to opam install after the package path. required: false default: --deps-only --with-test + restore-earlier: + description: >- + Whether a pull request may restore and update the environment of + earlier dependencies when its own is not cached. A job that never does + resolves every change of the dependencies from scratch, as master will. + required: false + default: "true" outputs: setup_version: description: Version of ocaml/setup-ocaml used by this action. @@ -81,10 +88,16 @@ runs: dependency_flags_key="${DEPENDENCY_FLAGS//--/}" dependency_flags_key="${dependency_flags_key// /_}" dependency_hash="${{ hashFiles('*.opam') }}" - key="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-$dependency_hash" - echo "value=${key//,/-}" >> "$GITHUB_OUTPUT" + prefix="$CACHE_PREFIX-$OS-3.9.0-$COMPILER-$dependency_flags_key-" + echo "prefix=${prefix//,/-}" >> "$GITHUB_OUTPUT" + echo "value=${prefix//,/-}$dependency_hash" >> "$GITHUB_OUTPUT" echo "setup-version=3.9.0" >> "$GITHUB_OUTPUT" + # When a pull request changes the dependencies, the environment of the + # previous ones is restored and updated, which takes minutes less than + # building it anew. Pushes always build a missing environment anew: what + # they save is resolved from scratch, so packages dropped from the opam + # files, pins and earlier upgrades don't accumulate in it. - name: Restore OPAM environment id: cache uses: actions/cache/restore@v6 @@ -97,22 +110,25 @@ runs: C:\cygwin C:\.opam key: ${{ steps.cache-key.outputs.value }} + restore-keys: ${{ github.event_name == 'pull_request' && inputs.restore-earlier == 'true' && steps.cache-key.outputs.prefix || '' }} - name: Use OCaml ${{ inputs.compiler }} - if: steps.cache.outputs.cache-hit != 'true' + if: steps.cache.outputs.cache-matched-key == '' uses: ocaml/setup-ocaml@v3.9.0 with: ocaml-compiler: ${{ inputs.compiler }} # Use the HTTP repository to prevent intermittent CI setup failures: # the Git-backed default can race with detached Git maintenance while # OPAM scans it (https://github.com/ocaml/opam/issues/7031). + # A restored environment keeps its repositories, so a change here + # needs a new cache-prefix. opam-repositories: | default: https://opam.ocaml.org opam-pin: false cache: false - name: Get OPAM executable path - if: steps.cache.outputs.cache-hit != 'true' + if: steps.cache.outputs.cache-matched-key == '' uses: actions/github-script@v9 with: script: | @@ -123,40 +139,8 @@ runs: await fs.writeFile('.opam-path', opamPath, 'utf-8'); console.log('stored path to .opam-path'); - - name: Install OPAM dependencies - if: steps.cache.outputs.cache-hit != 'true' - shell: bash - env: - DEPENDENCY_FLAGS: ${{ inputs.dependency-flags }} - run: | - read -ra dependency_flags <<< "$DEPENDENCY_FLAGS" - if [[ "$RUNNER_OS" == "Windows" ]]; then - export AR=x86_64-w64-mingw32-ar - export RANLIB=x86_64-w64-mingw32-ranlib - export STRIP=x86_64-w64-mingw32-strip - export NM=x86_64-w64-mingw32-nm - export DLLTOOL=x86_64-w64-mingw32-dlltool - export OBJDUMP=x86_64-w64-mingw32-objdump - fi - opam install . "${dependency_flags[@]}" - - - name: Cache OPAM environment - # Caches created by pull_request runs are scoped to that PR's merge ref - # and cannot seed other PRs or branches, so only push runs save them. - if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'push' - uses: actions/cache/save@v6 - with: - path: | - ${{ runner.tool_cache }}/opam - ~/.opam - _opam - .opam-path - C:\cygwin - C:\.opam - key: ${{ steps.cache-key.outputs.value }} - - name: Use cached OPAM environment - if: steps.cache.outputs.cache-hit == 'true' + if: steps.cache.outputs.cache-matched-key != '' shell: bash run: | echo "OPAMCOLOR=always" >> "$GITHUB_ENV" @@ -190,3 +174,61 @@ runs: echo "CYGWIN=winsymlinks:native" >> "$GITHUB_ENV" echo "BASH_ENV=C:\\cygwin\\bash_env" >> "$GITHUB_ENV" fi + + - name: Install OPAM dependencies + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + DEPENDENCY_FLAGS: ${{ inputs.dependency-flags }} + RESTORED: ${{ steps.cache.outputs.cache-matched-key != '' }} + run: | + read -ra dependency_flags <<< "$DEPENDENCY_FLAGS" + if [[ "$RUNNER_OS" != "Windows" ]]; then + # setup-linux-dependencies unpacks musl-tools without dpkg, and + # Binaryen is installed above, so opam needn't install either. + # Without this, a restored environment takes both for removed and + # rebuilds the whole switch. + opam option --global 'depext-bypass=["binaryen" "musl-tools"]' + fi + if [[ "$RESTORED" == "true" ]]; then + # Pins of earlier dependencies would hide every other version of + # their packages; the install pins what the opam files still pin. + for package in $(opam pin list --short | tr -d '\r'); do + opam unpin --no-action "$package" + done + # The restored environment may predate packages the dependencies + # now need. The update leaves the installed packages as they are, + # as on an exact cache hit. + opam update --repositories + opam reinstall --forget-pending + if [[ "$RUNNER_OS" == "Windows" ]]; then + # Where ocaml/setup-ocaml points it; the cached environment's + # setting names a file that doesn't exist. + export BASH_ENV='C:\.opam\.cygwin\root\bash_env' + fi + fi + if [[ "$RUNNER_OS" == "Windows" ]]; then + export AR=x86_64-w64-mingw32-ar + export RANLIB=x86_64-w64-mingw32-ranlib + export STRIP=x86_64-w64-mingw32-strip + export NM=x86_64-w64-mingw32-nm + export DLLTOOL=x86_64-w64-mingw32-dlltool + export OBJDUMP=x86_64-w64-mingw32-objdump + fi + opam install . "${dependency_flags[@]}" + + - name: Cache OPAM environment + # Caches created by pull_request runs are scoped to that PR's merge ref + # and cannot seed other PRs or branches, so only push runs save them. + # Only environments resolved from scratch are saved. + if: steps.cache.outputs.cache-matched-key == '' && github.event_name == 'push' + uses: actions/cache/save@v6 + with: + path: | + ${{ runner.tool_cache }}/opam + ~/.opam + _opam + .opam-path + C:\cygwin + C:\.opam + key: ${{ steps.cache-key.outputs.value }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 229bcecc52..15310e30a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,14 +99,21 @@ jobs: compiler: ${{ matrix.ocaml_compiler }} os: ${{ matrix.os }} cache-prefix: opam-env-v9 + # This job resolves every change of the dependencies from scratch, + # as master will after the merge. + restore-earlier: ${{ !contains(matrix.ocaml_compiler, '5.0.0') }} + # The key doesn't change with the dependencies: dune rebuilds what they + # change. Master deletes the state and saves it again. - name: Compiler build state key id: compiler-build-state-key shell: bash run: | - echo "value=compiler-build-state-v1-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v1-${{ matrix.os }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ matrix.ocaml_compiler }}" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" + # The prefix also restores the states saved under the earlier keys, + # which ended in a hash of the opam files. - name: Restore compiler build state if: github.base_ref == 'master' || github.ref == 'refs/heads/master' id: compiler-build-state @@ -117,6 +124,7 @@ jobs: ~/.cache/dune _build key: ${{ steps.compiler-build-state-key.outputs.value }} + restore-keys: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build compiler run: opam exec -- dune build --display quiet @@ -461,7 +469,7 @@ jobs: - name: Compiler build state key id: compiler-build-state-key run: | - echo "value=compiler-build-state-v1-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER-${{ hashFiles('*.opam') }}" \ + echo "value=compiler-build-state-v1-ubuntu-26.04-${{ steps.setup_ocaml.outputs.setup_version }}-$OCAML_COMPILER" \ | sed 's/,/-/g' >> "$GITHUB_OUTPUT" - name: Restore compiler build state @@ -474,6 +482,7 @@ jobs: ~/.cache/dune _build key: ${{ steps.compiler-build-state-key.outputs.value }} + restore-keys: ${{ steps.compiler-build-state-key.outputs.value }} - name: Build the compiler run: opam exec -- dune build --display quiet diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 606a1b44c2..63e3a51510 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -49,13 +49,17 @@ jobs: # --- Coverage build cache -------------------------------------------- # The bisect_ppx-instrumented dune build is a separate artifact from - # the main CI build, so it gets its own key. Only restore/save on - # master pushes — PRs share the master cache to stay fast but won't - # poison it. + # the main CI build, so it gets its own key. Only save on master + # pushes — PRs share the master cache to stay fast but won't poison + # it. The key changes with neither the compiler nor the dependencies: + # dune rebuilds what they change, and master deletes the state and + # saves it again, so it keeps one. - name: Coverage build state key id: coverage-build-state-key - run: echo "value=coverage-build-state-v1-${{ env.OS }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ env.OCAML_COMPILER }}-${{ hashFiles('*.opam', 'compiler/**', 'dune-project') }}" >> $GITHUB_OUTPUT + run: echo "value=coverage-build-state-v1-${{ env.OS }}-${{ steps.setup_ocaml.outputs.setup_version }}-${{ env.OCAML_COMPILER }}" >> "$GITHUB_OUTPUT" + # The prefix also restores the states saved under the earlier keys, + # which ended in a hash of the sources. - name: Restore coverage build state if: github.base_ref == 'master' || github.ref == 'refs/heads/master' id: coverage-build-state @@ -65,6 +69,8 @@ jobs: ~/.cache/dune _build key: ${{ steps.coverage-build-state-key.outputs.value }} + restore-keys: ${{ steps.coverage-build-state-key.outputs.value }} + - name: Run coverage run: opam exec -- make coverage