diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3804271814..3a3803a59e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1041,8 +1041,24 @@ jobs: # simply never created. The literal after `||` is the full roster, read # when the plan job published nothing (it died or timed out), and the gate # reads this same expression, so the two cannot count different rosters. + # + # ⭐ FAIL-FAST IN THE MERGE QUEUE ONLY (#31517, ruling record 6730013883). + # On a `merge_group` run the first shard to go red cancels its siblings, + # so `Test Core` reads red when that shard ends instead of when the + # slowest shard ends, and the queue ejects the entry and rebuilds the + # entries behind it that much sooner. On every other event the + # expression is false and every shard runs to its end: the pull_request + # run is where an author reads every red shard in one round (#23177 + # ruling B, item A1, comment 6139629628), and push and schedule runs stay + # complete. The queue is the exception because its job is a fast + # mergeable-or-not answer, and the PR run of the same diff already + # showed every red. It is the matrix's own `fail-fast`, ⛔ never a + # run-level cancel: a real red dominates the matrix result (`failure`, + # never `cancelled`, measured on run 30271824408, a fail-fast matrix), + # and `Test Core` reads `failure` as a veto, whereas a matrix cancelled + # with zero attestations passes its aggregate (#3668). strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: ${{ fromJSON(needs.test-plan.outputs.shards || '[1, 2, 3, 4, 5, 6, 7, 8]') }} @@ -1580,6 +1596,11 @@ jobs: # non-zero exit, and the step exits with it, so the shard is red # exactly when it was before — and now names every red package of the # shard in one run instead of the first one turbo reached. + # A1 holds inside a shard on every event. Across shards the merge + # queue is the exception (#31517, ruling record 6730013883): there + # the matrix is fail-fast, so the first red shard cancels the + # others, and the queue run names that shard's reds only. The + # pull_request run of the same diff names every shard's. # # Measured before this was written: a three-package probe (turbo # 2.11.5, `a#test` exits 1, `c` depends on `a`) ran `b` and `c` to @@ -2716,8 +2737,13 @@ jobs: timeout-minutes: 30 permissions: contents: read + # Fail-fast on `merge_group` only, as on Test Core (#31517, ruling record + # 6730013883; the reasons are on the `test` job's strategy). Push and + # schedule, the other events this matrix runs on, run all three shards to + # their end. A red shard dominates the matrix result over the siblings it + # cancels, and `Dogfood Regression Gate` reads `failure` as a veto. strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: [1, 2, 3] @@ -3083,7 +3109,11 @@ jobs: # 30271824408, a fail-fast matrix with one real failure + one cancelled # sibling): a real shard failure DOMINATES the aggregate — it reads # "failure", never "cancelled" — so passing `cancelled` masks no - # regression. That dominance is what does NOT hold for `abandoned`, which + # regression. That run is the merge queue's own shape since #31517: the + # dogfood matrix is fail-fast on `merge_group`, and the sibling a red + # shard cancels records "The strategy configuration was canceled + # because …" on its check run, which the cause read classifies as no + # supersession: a partially attested matrix it cut short reads red. That dominance is what does NOT hold for `abandoned`, which # is why `abandoned` gets counting rather than a place in a word list. name: Dogfood Regression Gate needs: [dogfood, dogfood-verify, filter, queue-verdict] @@ -4147,8 +4177,15 @@ jobs: timeout-minutes: 30 permissions: contents: read + # Fail-fast on `merge_group` only, as on Test Core (#31517, ruling record + # 6730013883; the reasons are on the `test` job's strategy). Every other + # event runs all sixteen shards to their end. The `Test UI` aggregate + # passes only `success` or a skip `filter` explains, so the matrix of a + # red shard and the siblings it cancelled reads red there. A fail-fast queue run is red, so a push never + # borrows it: `queue-verdict` takes only a `success` queue run, and + # `ui_full_green` counts only shards that concluded `success`. strategy: - fail-fast: false + fail-fast: ${{ github.event_name == 'merge_group' }} matrix: shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] @@ -4373,7 +4410,9 @@ jobs: # That is #4928's skipped-only-when-filter-succeeded guard, and it makes a # queue build or PR that touches no ui path read green here the way Lint # UI and Build & E2E UI read `skipped` there. Everything else is red: - # `failure`; `cancelled`, which is also what a shard killed by its own + # `failure`, which is also what the matrix reads when a red shard on a + # `merge_group` run cancelled its siblings by fail-fast (#31517); + # `cancelled`, which is also what a shard killed by its own # `timeout-minutes` reads (on a superseded push this paints the OLD sha # red, which no required check reads); a `skipped` that `filter` does not # explain; and any lifecycle value such as the `abandoned` test-gate met diff --git a/scripts/check-shard-attestation.mjs b/scripts/check-shard-attestation.mjs index d1614f3cd6f..53a60f370fd 100644 --- a/scripts/check-shard-attestation.mjs +++ b/scripts/check-shard-attestation.mjs @@ -80,6 +80,13 @@ * `cancelled`. So a superseded matrix masks no regression, and demanding * credentials there would paint the false red on the superseded SHA that * #3668 removed. + * Since #31517 (ruling record 6730013883) the merge queue's `test` and + * `dogfood` matrices are fail-fast, which is run 30271824408's own shape: + * the red shard's `failure` dominates the leg, the leg's declared negative + * vetoes the gate, and the shards the strategy cancelled are missing from + * the count. Such a shard records `The strategy configuration was + * canceled because "…" failed` on its check run, which + * `classifyCancelAnnotations` reads as `other`, never a supersession. * ⛔ That argument covers a shard that RAN AND FAILED. It does not cover a * shard killed while still executing, which produces no verdict at all — * there is no `failure` to dominate (#16157, run 34007386254: `Test Core @@ -1740,11 +1747,12 @@ const SELF_TEST_BATTERIES = Object.freeze({ '#26084 a planned roster: only the shards the plan started are counted': 16, '#26084 the static guard holds the planned matrix and the gate to one expression': 10, '#31519 a borrowed push: the gate skip and the seed-only credential guard': 8, + '#31517 merge-queue fail-fast: one shard red, the rest cancelled by the strategy': 8, }); // DELETING an entry silences that battery's floor exactly as effectively as // zeroing it, so the roster's own size is pinned too. -const SELF_TEST_BATTERY_FLOOR = 25; +const SELF_TEST_BATTERY_FLOOR = 26; // The key an assertion is filed under when no battery is open. It is not a // declared battery, so it reds by the same set difference rather than silently @@ -2293,6 +2301,40 @@ async function selfTest() { "#22653 control: the single-leg path keeps #22606's verdict and words — a killed CLI leg is red beside a superseded matrix shard", ); + // ── #31517: the merge queue's matrices are fail-fast ───────────────────── + // On `merge_group` the `test` and `dogfood` matrices stop their remaining + // shards once one is red (ruling record 6730013883). The shape is run + // 30271824408's: the red shard concludes `failure`, the shards the strategy + // cancelled conclude `cancelled` and attest nothing, and the matrix result + // reads `failure`. The annotation below is that run's cancelled shard (check + // run 89995981782), read from the check-run annotations endpoint. + battery('#31517 merge-queue fail-fast: one shard red, the rest cancelled by the strategy'); + const FAIL_FAST_ANNOTATIONS = [annotation('The strategy configuration was canceled because "matrix._1" failed'), annotation('The operation was canceled.')]; + const failFast = (ids) => testCore(ids, undefined, { result: 'failure' }); + const fastShardsFirst = failFast([1, 3].map((n) => attest('test', n, 6))); + assert(!fastShardsFirst.ok, '#31517: shard 2 red, 1 and 3 attested before it, 4-6 cancelled by fail-fast ⇒ red'); + assert( + fastShardsFirst.errors.some((e) => e.includes("reported result 'failure'")) && + fastShardsFirst.errors.some((e) => [2, 4, 5, 6].every((n) => e.includes(`test-${n}-of-6`))), + '#31517: …by the declared negative AND by the count, which names the red shard and every shard it cancelled', + ); + assert(!failFast([]).ok, '#31517: the first shard red before any sibling attested, all cancelled ⇒ red — `failure` is never the zero-attestation #3668 pass'); + assert( + shardsNeedingCancelCause(testLegs('failure'), new Map([1, 3].map((n) => attest('test', n, 6)))).length === 0, + '#31517: a fail-fast matrix reads `failure`, so the gate asks no cancel cause — no API read can turn it green', + ); + assert(classifyCancelAnnotations(FAIL_FAST_ANNOTATIONS).kind === 'other', "#31517: a shard cancelled by its matrix's fail-fast is never read as a supersession"); + const ifItEverReadCancelled = testCore(only5, new Map([1, 2, 3, 4, 6].map((n) => [attestationId('test', n, 6), classifyCancelAnnotations(FAIL_FAST_ANNOTATIONS)]))); + assert( + !ifItEverReadCancelled.ok && ifItEverReadCancelled.errors.some((e) => e.includes('The strategy configuration was canceled')), + '#31517: were the matrix ever to read `cancelled` after a sibling attested, the fail-fast cause still reds it, quoting the record', + ); + assert( + !judge({ gate: 'Test Core', legs: [{ job: 'test', total: 8, result: 'failure', shards: [2, 5, 7], planResult: 'success' }], filterResult: 'success', present: new Map([attest('test', 5, 8)]), runId: '99' }).ok, + '#31517 × #26084: a planned roster cut short by fail-fast ⇒ red', + ); + assert(!dogfood('failure', 'success', [fullDogfood[0], fullDogfood[3]]).ok, '#31517: dogfood 2/3 red, 3/3 cancelled by fail-fast, the CLI leg green ⇒ the Dogfood Regression Gate reads red'); + // ── #22653: the per-shard cause read itself, offline ───────────────────── battery('#22653 the cause read, offline: one jobs list, then one annotations read per unattested shard'); const CI_TEXT = readFileSync(join(scriptRepoRoot(), '.github', 'workflows', 'ci.yml'), 'utf8'); diff --git a/scripts/ci-shard-matrix.mjs b/scripts/ci-shard-matrix.mjs index 48ada86067a..60b87708e1b 100644 --- a/scripts/ci-shard-matrix.mjs +++ b/scripts/ci-shard-matrix.mjs @@ -217,7 +217,9 @@ function selfTest() { check(`${label} ⇒ refused naming '${needle}' (got ${JSON.stringify(message)})`, message !== null && message.includes(needle) && message.includes(`job '${job}'`)); }; const workflow = (body) => `on: push\njobs:\n${body}`; - const matrixJob = (id, shardLine, extra = '') => ` ${id}:\n runs-on: ubuntu-latest\n${extra} strategy:\n fail-fast: false\n matrix:\n ${shardLine}\n steps:\n - run: echo\n`; + // The strategy spells ci.yml's sharded jobs' `fail-fast` (merge-queue only, #31517), so the + // reader is exercised over an expression line between `strategy:` and `matrix:`. + const matrixJob = (id, shardLine, extra = '') => ` ${id}:\n runs-on: ubuntu-latest\n${extra} strategy:\n fail-fast: \${{ github.event_name == 'merge_group' }}\n matrix:\n ${shardLine}\n steps:\n - run: echo\n`; battery('the live ci.yml'); const live = readFileSync(join(REPO_ROOT, CI_WORKFLOW), 'utf8');