Shard the mutation analysis across a matrix #21
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: mutation | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| env: | |
| BUNDLE_WITHOUT: "benchmark" | |
| # Subjects per runner. Mutant costs anywhere from half a second to twenty | |
| # seconds a subject, depending on how many examples the subject drags in, so | |
| # 50 is the size that keeps even a shard of the slow ones inside the timeout | |
| # below while leaving an ordinary pull request on a single runner. | |
| SUBJECTS_PER_SHARD: 50 | |
| # A ceiling on the fan-out, so a formatting pass over the whole tree grows | |
| # the shards rather than the runners. Mutant selects a subject whose line | |
| # range the diff touches, and a comment removed above a method moves every | |
| # method below it, so such a change can select most of the library. | |
| MAX_SHARDS: 24 | |
| jobs: | |
| plan: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| base: ${{ steps.plan.outputs.base }} | |
| count: ${{ steps.plan.outputs.count }} | |
| shards: ${{ steps.plan.outputs.shards }} | |
| total: ${{ steps.plan.outputs.total }} | |
| steps: | |
| # Mutant selects subjects by diffing against a revision, so the | |
| # revision has to be in the clone: a shallow one has nothing to | |
| # compare against. | |
| - uses: actions/checkout@v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ruby/setup-ruby@v1.321.0 | |
| with: | |
| ruby-version: ruby | |
| rubygems: default | |
| bundler-cache: true | |
| - name: Fetch the base branch | |
| if: github.event_name == 'pull_request' | |
| run: git fetch --no-tags origin ${{ github.base_ref }} | |
| # A pull request is diffed against its base branch. A push is diffed | |
| # against the head it replaced, so a multi-commit push is diffed as a | |
| # whole. A force push can leave that head unreachable, and then the diff | |
| # falls back to the tip commit alone, which covers the amend-and-push | |
| # case that produces most force pushes here. | |
| - name: Plan the shards | |
| id: plan | |
| run: | | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| base="origin/${{ github.base_ref }}" | |
| else | |
| base="${{ github.event.before }}" | |
| if [ "$base" = "0000000000000000000000000000000000000000" ]; then | |
| base="HEAD~1" | |
| elif ! git cat-file -e "$base^{commit}" 2>/dev/null; then | |
| git fetch --no-tags origin "$base" 2>/dev/null || base="HEAD~1" | |
| fi | |
| fi | |
| base=$(git rev-parse "$base^{commit}") | |
| echo "Comparing against $base" | |
| count=$(bundle exec rake -s "mutant:subjects[$base]" | grep -c . || true) | |
| if [ "$count" -eq 0 ]; then | |
| total=0 | |
| else | |
| total=$(( (count + SUBJECTS_PER_SHARD - 1) / SUBJECTS_PER_SHARD )) | |
| if [ "$total" -gt "$MAX_SHARDS" ]; then total="$MAX_SHARDS"; fi | |
| fi | |
| echo "$count subject(s) across $total shard(s)" | |
| { | |
| echo "base=$base" | |
| echo "count=$count" | |
| echo "total=$total" | |
| echo "shards=$(jq -nc --argjson n "$total" '[range($n)]')" | |
| } >> "$GITHUB_OUTPUT" | |
| # A change that touches no library code selects nothing and never reaches | |
| # here, which is what a docs- or spec-only change should do. | |
| mutate: | |
| needs: plan | |
| if: needs.plan.outputs.count != '0' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| # Every shard reports its own survivors. Stopping the others on the | |
| # first failure would hide the rest of them. | |
| fail-fast: false | |
| matrix: | |
| shard: ${{ fromJSON(needs.plan.outputs.shards) }} | |
| steps: | |
| - uses: actions/checkout@v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - uses: ruby/setup-ruby@v1.321.0 | |
| with: | |
| ruby-version: ruby | |
| rubygems: default | |
| bundler-cache: true | |
| - name: Make the base revision available | |
| run: | | |
| git cat-file -e "${{ needs.plan.outputs.base }}^{commit}" 2>/dev/null || | |
| git fetch --no-tags origin "${{ needs.plan.outputs.base }}" | |
| - name: Analyse this shard of the mutations the change touches | |
| run: | | |
| bundle exec rake \ | |
| "mutant:shard[${{ needs.plan.outputs.base }},${{ matrix.shard }},${{ needs.plan.outputs.total }}]" |