From 6903a73de37ef24aa0a4888a4e38573d4ba48874 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 01:32:41 +0000 Subject: [PATCH] ci: fail fast on the test, test-ui and dogfood matrices in the merge queue only `strategy.fail-fast: ${{ github.event_name == 'merge_group' }}` on the three matrices: a red queue shard cancels its siblings, so the aggregate reads red when that shard ends rather than when the slowest one does. Pull request, push and schedule runs keep every shard running to its end. The A1 comments say the queue is the exception and why. check-shard-attestation's self-test gains the fail-fast shape (one shard red, the rest cancelled by the strategy, the measured cancel annotation of run 30271824408), and ci-shard-matrix's fixture spells the new strategy. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QsR2jSkaG5iii6WC7w4qSX --- .github/workflows/ci.yml | 49 ++++++++++++++++++++++++++--- scripts/check-shard-attestation.mjs | 44 +++++++++++++++++++++++++- scripts/ci-shard-matrix.mjs | 4 ++- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3804271814..3a3803a59e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1041,8 +1041,24 @@ jobs: # simply never created. The literal after `||` is the full roster, read # when the plan job published nothing (it died or timed out), and the gate # reads this same expression, so the two cannot count different rosters. + # + # ⭐ FAIL-FAST IN THE MERGE QUEUE ONLY (#31517, ruling record 6730013883). + # On a `merge_group` run the first shard to go red cancels its siblings, + # so `Test Core` reads red when that shard ends instead of when the + # slowest shard ends, and the queue ejects the entry and rebuilds the + # entries behind it that much sooner. On every other event the + # expression is false and every shard runs to its end: the pull_request + # run is where an author reads every red shard in one round (#23177 + # ruling B, item A1, comment 6139629628), and push and schedule runs stay + # complete. The queue is the exception because its job is a fast + # mergeable-or-not answer, and the PR run of the same diff already + # showed every red. It is the matrix's own `fail-fast`, ⛔ never a + # run-level cancel: a real red dominates the matrix result (`failure`, + # never `cancelled`, measured on run 30271824408, a fail-fast matrix), + # and `Test Core` reads `failure` as a veto, whereas a matrix cancelled + # with zero attestations passes its aggregate (#3668). strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: ${{ fromJSON(needs.test-plan.outputs.shards || '[1, 2, 3, 4, 5, 6, 7, 8]') }} @@ -1580,6 +1596,11 @@ jobs: # non-zero exit, and the step exits with it, so the shard is red # exactly when it was before — and now names every red package of the # shard in one run instead of the first one turbo reached. + # A1 holds inside a shard on every event. Across shards the merge + # queue is the exception (#31517, ruling record 6730013883): there + # the matrix is fail-fast, so the first red shard cancels the + # others, and the queue run names that shard's reds only. The + # pull_request run of the same diff names every shard's. # # Measured before this was written: a three-package probe (turbo # 2.11.5, `a#test` exits 1, `c` depends on `a`) ran `b` and `c` to @@ -2716,8 +2737,13 @@ jobs: timeout-minutes: 30 permissions: contents: read + # Fail-fast on `merge_group` only, as on Test Core (#31517, ruling record + # 6730013883; the reasons are on the `test` job's strategy). Push and + # schedule, the other events this matrix runs on, run all three shards to + # their end. A red shard dominates the matrix result over the siblings it + # cancels, and `Dogfood Regression Gate` reads `failure` as a veto. strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: [1, 2, 3] @@ -3083,7 +3109,11 @@ jobs: # 30271824408, a fail-fast matrix with one real failure + one cancelled # sibling): a real shard failure DOMINATES the aggregate — it reads # "failure", never "cancelled" — so passing `cancelled` masks no - # regression. That dominance is what does NOT hold for `abandoned`, which + # regression. That run is the merge queue's own shape since #31517: the + # dogfood matrix is fail-fast on `merge_group`, and the sibling a red + # shard cancels records "The strategy configuration was canceled + # because …" on its check run, which the cause read classifies as no + # supersession: a partially attested matrix it cut short reads red. That dominance is what does NOT hold for `abandoned`, which # is why `abandoned` gets counting rather than a place in a word list. name: Dogfood Regression Gate needs: [dogfood, dogfood-verify, filter, queue-verdict] @@ -4147,8 +4177,15 @@ jobs: timeout-minutes: 30 permissions: contents: read + # Fail-fast on `merge_group` only, as on Test Core (#31517, ruling record + # 6730013883; the reasons are on the `test` job's strategy). Every other + # event runs all sixteen shards to their end. The `Test UI` aggregate + # passes only `success` or a skip `filter` explains, so the matrix of a + # red shard and the siblings it cancelled reads red there. A fail-fast queue run is red, so a push never + # borrows it: `queue-verdict` takes only a `success` queue run, and + # `ui_full_green` counts only shards that concluded `success`. strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] @@ -4373,7 +4410,9 @@ jobs: # That is #4928's skipped-only-when-filter-succeeded guard, and it makes a # queue build or PR that touches no ui path read green here the way Lint # UI and Build & E2E UI read `skipped` there. Everything else is red: - # `failure`; `cancelled`, which is also what a shard killed by its own + # `failure`, which is also what the matrix reads when a red shard on a + # `merge_group` run cancelled its siblings by fail-fast (#31517); + # `cancelled`, which is also what a shard killed by its own # `timeout-minutes` reads (on a superseded push this paints the OLD sha # red, which no required check reads); a `skipped` that `filter` does not # explain; and any lifecycle value such as the `abandoned` test-gate met diff --git a/scripts/check-shard-attestation.mjs b/scripts/check-shard-attestation.mjs index d1614f3cd6f..53a60f370fd 100644 --- a/scripts/check-shard-attestation.mjs +++ b/scripts/check-shard-attestation.mjs @@ -80,6 +80,13 @@ * `cancelled`. So a superseded matrix masks no regression, and demanding * credentials there would paint the false red on the superseded SHA that * #3668 removed. + * Since #31517 (ruling record 6730013883) the merge queue's `test` and + * `dogfood` matrices are fail-fast, which is run 30271824408's own shape: + * the red shard's `failure` dominates the leg, the leg's declared negative + * vetoes the gate, and the shards the strategy cancelled are missing from + * the count. Such a shard records `The strategy configuration was + * canceled because "…" failed` on its check run, which + * `classifyCancelAnnotations` reads as `other`, never a supersession. * ⛔ That argument covers a shard that RAN AND FAILED. It does not cover a * shard killed while still executing, which produces no verdict at all — * there is no `failure` to dominate (#16157, run 34007386254: `Test Core @@ -1740,11 +1747,12 @@ const SELF_TEST_BATTERIES = Object.freeze({ '#26084 a planned roster: only the shards the plan started are counted': 16, '#26084 the static guard holds the planned matrix and the gate to one expression': 10, '#31519 a borrowed push: the gate skip and the seed-only credential guard': 8, + '#31517 merge-queue fail-fast: one shard red, the rest cancelled by the strategy': 8, }); // DELETING an entry silences that battery's floor exactly as effectively as // zeroing it, so the roster's own size is pinned too. -const SELF_TEST_BATTERY_FLOOR = 25; +const SELF_TEST_BATTERY_FLOOR = 26; // The key an assertion is filed under when no battery is open. It is not a // declared battery, so it reds by the same set difference rather than silently @@ -2293,6 +2301,40 @@ async function selfTest() { "#22653 control: the single-leg path keeps #22606's verdict and words — a killed CLI leg is red beside a superseded matrix shard", ); + // ── #31517: the merge queue's matrices are fail-fast ───────────────────── + // On `merge_group` the `test` and `dogfood` matrices stop their remaining + // shards once one is red (ruling record 6730013883). The shape is run + // 30271824408's: the red shard concludes `failure`, the shards the strategy + // cancelled conclude `cancelled` and attest nothing, and the matrix result + // reads `failure`. The annotation below is that run's cancelled shard (check + // run 89995981782), read from the check-run annotations endpoint. + battery('#31517 merge-queue fail-fast: one shard red, the rest cancelled by the strategy'); + const FAIL_FAST_ANNOTATIONS = [annotation('The strategy configuration was canceled because "matrix._1" failed'), annotation('The operation was canceled.')]; + const failFast = (ids) => testCore(ids, undefined, { result: 'failure' }); + const fastShardsFirst = failFast([1, 3].map((n) => attest('test', n, 6))); + assert(!fastShardsFirst.ok, '#31517: shard 2 red, 1 and 3 attested before it, 4-6 cancelled by fail-fast ⇒ red'); + assert( + fastShardsFirst.errors.some((e) => e.includes("reported result 'failure'")) && + fastShardsFirst.errors.some((e) => [2, 4, 5, 6].every((n) => e.includes(`test-${n}-of-6`))), + '#31517: …by the declared negative AND by the count, which names the red shard and every shard it cancelled', + ); + assert(!failFast([]).ok, '#31517: the first shard red before any sibling attested, all cancelled ⇒ red — `failure` is never the zero-attestation #3668 pass'); + assert( + shardsNeedingCancelCause(testLegs('failure'), new Map([1, 3].map((n) => attest('test', n, 6)))).length === 0, + '#31517: a fail-fast matrix reads `failure`, so the gate asks no cancel cause — no API read can turn it green', + ); + assert(classifyCancelAnnotations(FAIL_FAST_ANNOTATIONS).kind === 'other', "#31517: a shard cancelled by its matrix's fail-fast is never read as a supersession"); + const ifItEverReadCancelled = testCore(only5, new Map([1, 2, 3, 4, 6].map((n) => [attestationId('test', n, 6), classifyCancelAnnotations(FAIL_FAST_ANNOTATIONS)]))); + assert( + !ifItEverReadCancelled.ok && ifItEverReadCancelled.errors.some((e) => e.includes('The strategy configuration was canceled')), + '#31517: were the matrix ever to read `cancelled` after a sibling attested, the fail-fast cause still reds it, quoting the record', + ); + assert( + !judge({ gate: 'Test Core', legs: [{ job: 'test', total: 8, result: 'failure', shards: [2, 5, 7], planResult: 'success' }], filterResult: 'success', present: new Map([attest('test', 5, 8)]), runId: '99' }).ok, + '#31517 × #26084: a planned roster cut short by fail-fast ⇒ red', + ); + assert(!dogfood('failure', 'success', [fullDogfood[0], fullDogfood[3]]).ok, '#31517: dogfood 2/3 red, 3/3 cancelled by fail-fast, the CLI leg green ⇒ the Dogfood Regression Gate reads red'); + // ── #22653: the per-shard cause read itself, offline ───────────────────── battery('#22653 the cause read, offline: one jobs list, then one annotations read per unattested shard'); const CI_TEXT = readFileSync(join(scriptRepoRoot(), '.github', 'workflows', 'ci.yml'), 'utf8'); diff --git a/scripts/ci-shard-matrix.mjs b/scripts/ci-shard-matrix.mjs index 48ada86067a..60b87708e1b 100644 --- a/scripts/ci-shard-matrix.mjs +++ b/scripts/ci-shard-matrix.mjs @@ -217,7 +217,9 @@ function selfTest() { check(`${label} ⇒ refused naming '${needle}' (got ${JSON.stringify(message)})`, message !== null && message.includes(needle) && message.includes(`job '${job}'`)); }; const workflow = (body) => `on: push\njobs:\n${body}`; - const matrixJob = (id, shardLine, extra = '') => ` ${id}:\n runs-on: ubuntu-latest\n${extra} strategy:\n fail-fast: false\n matrix:\n ${shardLine}\n steps:\n - run: echo\n`; + // The strategy spells ci.yml's sharded jobs' `fail-fast` (merge-queue only, #31517), so the + // reader is exercised over an expression line between `strategy:` and `matrix:`. + const matrixJob = (id, shardLine, extra = '') => ` ${id}:\n runs-on: ubuntu-latest\n${extra} strategy:\n fail-fast: \${{ github.event_name == 'merge_group' }}\n matrix:\n ${shardLine}\n steps:\n - run: echo\n`; battery('the live ci.yml'); const live = readFileSync(join(REPO_ROOT, CI_WORKFLOW), 'utf8');