diff --git a/.github/actions/classify-changes/action.yml b/.github/actions/classify-changes/action.yml
new file mode 100644
index 0000000000..cad30a10c7
--- /dev/null
+++ b/.github/actions/classify-changes/action.yml
@@ -0,0 +1,56 @@
+name: "Classify changed files"
+
+description: "Classifies a pull request change range as documentation-only or full validation. The classification fails safe: every uncertain input selects full validation."
+
+inputs:
+ event-name:
+ description: "GitHub event name that drives skip eligibility"
+ required: true
+ head-sha:
+ description: "Head commit SHA of the classification range"
+ required: true
+ force-full-label:
+ description: "When true, force full validation because the ci:full-validation label is present"
+ required: false
+ default: "false"
+ force-full-variable:
+ description: "When true, force full validation because the ZOO_CODE_REQUIRE_FULL_VALIDATION repository variable is true"
+ required: false
+ default: "false"
+
+outputs:
+ category:
+ description: "docs-only or full"
+ value: ${{ steps.classify.outputs.category }}
+ reason:
+ description: "Human-readable classification reason"
+ value: ${{ steps.classify.outputs.reason }}
+ reason-code:
+ description: "Stable classification reason code"
+ value: ${{ steps.classify.outputs.reason-code }}
+ changed-files:
+ description: "Number of changed files in the classification range"
+ value: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint:
+ description: "Fingerprint of the active documentation-only allowlist"
+ value: ${{ steps.classify.outputs.allowlist-fingerprint }}
+
+runs:
+ using: "composite"
+ steps:
+ - id: classify
+ shell: bash
+ run: |
+ HEAD_SHA="${{ inputs.head-sha }}"
+ FLAGS=()
+ if [[ "${{ inputs.force-full-label }}" == "true" ]]; then
+ FLAGS+=("--force-full-label")
+ fi
+ if [[ "${{ inputs.force-full-variable }}" == "true" ]]; then
+ FLAGS+=("--force-full-variable")
+ fi
+ # The pull_request checkout is the synthetic merge commit, so its
+ # first parent is the base branch tip that was actually merged
+ # into. A failed resolution fails the step and fails safe.
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-classification.mjs ci --event "${{ inputs.event-name }}" --head "$HEAD_SHA" --base "$BASE_SHA" "${FLAGS[@]}"
diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml
index 42b94efa3d..35287a98eb 100644
--- a/.github/workflows/code-qa.yml
+++ b/.github/workflows/code-qa.yml
@@ -5,7 +5,7 @@ on:
push:
branches: [main]
pull_request:
- types: [opened, reopened, ready_for_review, synchronize]
+ types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled]
merge_group:
types: [checks_requested]
@@ -20,6 +20,37 @@ permissions:
contents: read
jobs:
+ classify:
+ name: classify (code-qa)
+ # Classifies the pull request change range before any validation
+ # runs. Only an explicit documentation-only allowlist match can
+ # skip expensive jobs, and only for pull_request events; a failed
+ # or missing classification fails safe to full validation. The
+ # validation-gate job below audits this decision on every run.
+ runs-on: ubuntu-latest
+ outputs:
+ category: ${{ steps.classify.outputs.category }}
+ reason: ${{ steps.classify.outputs.reason }}
+ reason-code: ${{ steps.classify.outputs.reason-code }}
+ changed-files: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Classification resolves the merge base from history.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Classify changed files
+ id: classify
+ uses: ./.github/actions/classify-changes
+ with:
+ event-name: ${{ github.event_name }}
+ head-sha: ${{ github.sha }}
+ force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+
dependency-review:
runs-on: ubuntu-latest
# Only meaningful for PRs — validates the dependency diff of the pull
@@ -65,7 +96,30 @@ jobs:
exit 1
fi
+ format-check:
+ name: Formatting and validation policy tests
+ # Always runs, including documentation-only pull requests: Prettier
+ # checks every Markdown file, so formatting validation must never be
+ # reduced by classification. This job also validates the classifier
+ # and gate logic on every run.
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Setup Node.js and pnpm
+ uses: ./.github/actions/setup-node-pnpm
+ with:
+ install-args: "--frozen-lockfile"
+ - name: Check formatting
+ run: pnpm format:check
+ - name: Test change classification and validation gate
+ run: pnpm test:change-gate
+
check-translations:
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Everything else runs unchanged.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
steps:
- name: Checkout code
@@ -80,6 +134,8 @@ jobs:
run: node scripts/check-i18n-interpolation.js
knip:
+ # Intentionally ungated: the repository invariant test requires the
+ # knip job to run without conditions or allowed failure.
runs-on: ubuntu-latest
steps:
- name: Checkout code
@@ -96,42 +152,63 @@ jobs:
run: pnpm knip:production
compile:
+ needs: [classify, format-check]
+ # Start the required check even for docs-only changes.
+ if: ${{ !cancelled() }}
+ env:
+ RUN_VALIDATION: ${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }}
runs-on: ubuntu-latest
steps:
+ - name: Enforce formatting and validation policy tests
+ env:
+ FORMAT_RESULT: ${{ needs.format-check.result }}
+ run: test "$FORMAT_RESULT" = success
- name: Checkout code
+ if: env.RUN_VALIDATION == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js and pnpm
+ if: env.RUN_VALIDATION == 'true'
uses: ./.github/actions/setup-node-pnpm
with:
install-args: "--frozen-lockfile"
- - name: Check formatting
- run: pnpm format:check
- name: Lint
+ if: env.RUN_VALIDATION == 'true'
run: pnpm lint
- name: Check types
+ if: env.RUN_VALIDATION == 'true'
run: pnpm check-types
- name: Validate Code QA workflow
+ if: env.RUN_VALIDATION == 'true'
run: pnpm test:code-qa-ci
- name: Validate Solheim provider smoke workflow
+ if: env.RUN_VALIDATION == 'true'
run: pnpm test:solheim-smoke-ci
- name: Test Solheim provider smoke units
+ if: env.RUN_VALIDATION == 'true'
run: pnpm test:solheim-smoke:unit
- name: Check Solheim provider smoke types
+ if: env.RUN_VALIDATION == 'true'
run: pnpm solheim-smoke:check-types
- name: Test agent evals units
+ if: env.RUN_VALIDATION == 'true'
run: pnpm test:agent-evals:unit
- name: Check agent evals types
+ if: env.RUN_VALIDATION == 'true'
run: pnpm agent-evals:check-types
- name: Model-check task lifecycle protocols
+ if: env.RUN_VALIDATION == 'true'
run: pnpm lifecycle:model-check
- name: Model-check webview recovery
+ if: env.RUN_VALIDATION == 'true'
run: pnpm webview-recovery:check-types && pnpm webview-recovery:model-check
- name: Validate MCP OAuth integration
+ if: env.RUN_VALIDATION == 'true'
run: pnpm mcp:integration-check
build-vsix:
name: Build test VSIX
- if: github.event_name == 'pull_request'
+ needs: classify
+ if: ${{ !cancelled() && github.event_name == 'pull_request' && (needs.classify.result != 'success' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
steps:
- name: Checkout code
@@ -158,6 +235,11 @@ jobs:
unit-test:
name: platform-unit-test (${{ matrix.name }})
+ needs: classify
+ # Start the required check even for docs-only changes.
+ if: ${{ !cancelled() }}
+ env:
+ RUN_VALIDATION: ${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }}
runs-on: ${{ matrix.os }}
concurrency:
# Duplicate runs for one commit must not race the OS-specific Turbo cache save.
@@ -175,13 +257,20 @@ jobs:
codecov-flag: windows
collect-coverage: false
steps:
+ - name: Report documentation-only validation
+ if: env.RUN_VALIDATION == 'false'
+ shell: bash
+ run: echo "Documentation-only change; expensive platform validation steps skipped." >> "$GITHUB_STEP_SUMMARY"
- name: Checkout code
+ if: env.RUN_VALIDATION == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Node.js and pnpm
+ if: env.RUN_VALIDATION == 'true'
uses: ./.github/actions/setup-node-pnpm
with:
install-args: "--frozen-lockfile"
- name: Restore Turbo cache
+ if: env.RUN_VALIDATION == 'true'
id: turbo-cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
@@ -194,16 +283,16 @@ jobs:
# suites through the uninstrumented Turbo tasks. Ubuntu stays the
# authoritative coverage lane.
- name: Run non-extension package coverage
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm turbo run test:coverage --filter="!@roo-code/core" --filter="!zoo-code" --log-order grouped --output-logs new-only
- name: Run non-extension package tests
- if: ${{ !matrix.collect-coverage }}
+ if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}
run: pnpm turbo run test --filter="!@roo-code/core" --filter="!zoo-code" --log-order grouped --output-logs new-only
- name: Run extension coverage lanes
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm turbo run test:coverage:api test:coverage:core test:coverage:services test:coverage:misc test:coverage:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only
- name: Run extension test lanes
- if: ${{ !matrix.collect-coverage }}
+ if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}
run: pnpm turbo run test:api test:core test:services test:misc test:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only
# Runs outside Turbo so changes to .coderabbit.yaml,
# .github/workflows/label-pr-review-state.yml, and
@@ -211,26 +300,28 @@ jobs:
# sit outside the src/ package boundary and cannot be declared as
# Turbo inputs.
- name: Run workflow config tests
+ if: env.RUN_VALIDATION == 'true'
run: pnpm --dir src run test:workflow
- name: Verify extension coverage contract
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm --dir src run verify:coverage-contract
- name: Run extension dist smoke test
+ if: env.RUN_VALIDATION == 'true'
run: pnpm turbo run test:dist --filter="zoo-code" --log-order grouped --output-logs new-only
- name: Run core unit coverage
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm turbo run test:coverage:unit --filter="@roo-code/core" --log-order grouped --output-logs new-only
- name: Run core unit tests
- if: ${{ !matrix.collect-coverage }}
+ if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}
run: pnpm turbo run test:unit --filter="@roo-code/core" --log-order grouped --output-logs new-only
- name: Run core integration coverage
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm turbo run test:coverage:integration --filter="@roo-code/core" --log-order grouped --output-logs new-only
- name: Run core integration tests
- if: ${{ !matrix.collect-coverage }}
+ if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}
run: pnpm turbo run test:integration --filter="@roo-code/core" --log-order grouped --output-logs new-only
- name: Verify extension coverage reports
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: |
node src/scripts/verify-lcov.mjs src/coverage/api/lcov.info
node src/scripts/verify-lcov.mjs src/coverage/core/lcov.info
@@ -238,17 +329,17 @@ jobs:
node src/scripts/verify-lcov.mjs src/coverage/misc/lcov.info
node src/scripts/verify-lcov.mjs src/coverage/tree-sitter/lcov.info
- name: Merge extension coverage reports
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: |
mkdir -p src/coverage/merged
pnpm --dir src run merge:coverage
node src/scripts/verify-lcov.mjs src/coverage/merged/lcov.info
# Validate cache boundaries before publishing any new Turbo entries.
- name: Verify coverage cache inputs
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
run: pnpm --dir src run verify:coverage-cache-inputs
- name: Save Turbo cache
- if: steps.turbo-cache.outputs.cache-hit != 'true'
+ if: ${{ env.RUN_VALIDATION == 'true' && (steps.turbo-cache.outputs.cache-hit != 'true') }}
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: .turbo/cache
@@ -263,7 +354,7 @@ jobs:
# it. Core and webview reports retain their independent flags.
# See https://docs.codecov.com/docs/flags
- name: Upload non-core coverage to Codecov
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
files: >-
@@ -275,7 +366,7 @@ jobs:
flags: ${{ matrix.codecov-flag }}
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload webview JSDOM coverage to Codecov
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
files: webview-ui/coverage/lcov.info
@@ -283,7 +374,7 @@ jobs:
flags: webview-ui
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload core unit coverage to Codecov
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
files: packages/core/coverage/unit/lcov.info
@@ -291,7 +382,7 @@ jobs:
flags: ${{ matrix.codecov-flag }},core-unit
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload core integration coverage to Codecov
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
with:
files: packages/core/coverage/integration/lcov.info
@@ -299,7 +390,7 @@ jobs:
flags: ${{ matrix.codecov-flag }},core-integration
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload coverage reports to GitHub
- if: matrix.collect-coverage
+ if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: coverage-reports-${{ matrix.name }}
@@ -317,3 +408,50 @@ jobs:
packages/core/coverage/unit/lcov.info
packages/core/coverage/integration/lcov.info
retention-days: 7
+
+ validation-gate:
+ name: validation-gate (code-qa)
+ # Always-running audit gate for change-based validation. It
+ # rejects a missing or failed classification, validation jobs that
+ # did not run when full validation was required, jobs that ran when
+ # a documentation-only skip was expected, and a classifier replay
+ # that disagrees with the recorded classification. It also records
+ # the classification, the skipped work, runner minutes, and the
+ # required-check completion time in the job summary.
+ if: always()
+ needs:
+ - classify
+ - dependency-review
+ - invisible-chars
+ - format-check
+ - check-translations
+ - knip
+ - compile
+ - build-vsix
+ - unit-test
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The gate replays the classifier against the same range.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Run validation gate
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ EVENT_NAME: ${{ github.event_name }}
+ PR_DRAFT: ${{ github.event.pull_request.draft }}
+ RUN_ID: ${{ github.run_id }}
+ HEAD_SHA: ${{ github.sha }}
+ FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+ WATCHED_JOBS: dependency-review:pr-only:ungated,invisible-chars:always:ungated,format-check:always:ungated,check-translations:always:gated,knip:always:ungated,compile:always:ungated,build-vsix:pr-only:gated,unit-test:always:ungated
+ run: |
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
index 27bfd3dec1..4d1d294720 100644
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -4,6 +4,7 @@ on:
push:
branches: ["main"]
pull_request:
+ types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled]
schedule:
- cron: "24 19 * * 3"
@@ -19,8 +20,44 @@ permissions:
contents: read
jobs:
+ classify:
+ name: classify (codeql)
+ # Classifies the pull request change range before any validation
+ # runs. Only an explicit documentation-only allowlist match can
+ # skip expensive jobs, and only for pull_request events; a failed
+ # or missing classification fails safe to full validation. The
+ # validation-gate job below audits this decision on every run.
+ runs-on: ubuntu-latest
+ outputs:
+ category: ${{ steps.classify.outputs.category }}
+ reason: ${{ steps.classify.outputs.reason }}
+ reason-code: ${{ steps.classify.outputs.reason-code }}
+ changed-files: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Classification resolves the merge base from history.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Classify changed files
+ id: classify
+ uses: ./.github/actions/classify-changes
+ with:
+ event-name: ${{ github.event_name }}
+ head-sha: ${{ github.sha }}
+ force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+
analyze:
name: Analyze (${{ matrix.language }})
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue, push, and scheduled
+ # analysis always runs.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
# Runner size impacts CodeQL analysis time. To learn more, please see:
# - https://gh.io/recommended-hardware-resources-for-running-codeql
# - https://gh.io/supported-runners-and-hardware-resources
@@ -89,3 +126,43 @@ jobs:
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
with:
category: "/language:${{matrix.language}}"
+
+ validation-gate:
+ name: validation-gate (codeql)
+ # Always-running required gate for change-based validation. It
+ # rejects a missing or failed classification, validation jobs that
+ # did not run when full validation was required, jobs that ran when
+ # a documentation-only skip was expected, and a classifier replay
+ # that disagrees with the recorded classification. It also records
+ # the classification, the skipped work, runner minutes, and the
+ # required-check completion time in the job summary.
+ if: always()
+ needs:
+ - classify
+ - analyze
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The gate replays the classifier against the same range.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Run validation gate
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ EVENT_NAME: ${{ github.event_name }}
+ PR_DRAFT: ${{ github.event.pull_request.draft }}
+ RUN_ID: ${{ github.run_id }}
+ HEAD_SHA: ${{ github.sha }}
+ FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+ WATCHED_JOBS: analyze:always:gated
+ run: |
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 9c86fe9f77..0c20bcaaf3 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -3,7 +3,7 @@ name: E2E Tests (Mocked)
on:
workflow_dispatch:
pull_request:
- types: [opened, reopened, ready_for_review, synchronize]
+ types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled]
merge_group:
types: [checks_requested]
@@ -17,7 +17,42 @@ permissions:
contents: read
jobs:
+ classify:
+ name: classify (e2e)
+ # Classifies the pull request change range before any validation
+ # runs. Only an explicit documentation-only allowlist match can
+ # skip expensive jobs, and only for pull_request events; a failed
+ # or missing classification fails safe to full validation. The
+ # validation-gate job below audits this decision on every run.
+ runs-on: ubuntu-latest
+ outputs:
+ category: ${{ steps.classify.outputs.category }}
+ reason: ${{ steps.classify.outputs.reason }}
+ reason-code: ${{ steps.classify.outputs.reason-code }}
+ changed-files: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Classification resolves the merge base from history.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Classify changed files
+ id: classify
+ uses: ./.github/actions/classify-changes
+ with:
+ event-name: ${{ github.event_name }}
+ head-sha: ${{ github.sha }}
+ force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+
e2e-mock:
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue validation always runs.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -191,3 +226,43 @@ jobs:
with:
path: .cache/e2e-pass
key: ${{ steps.e2e-marker.outputs.cache-primary-key }}
+
+ validation-gate:
+ name: validation-gate (e2e)
+ # Always-running required gate for change-based validation. It
+ # rejects a missing or failed classification, validation jobs that
+ # did not run when full validation was required, jobs that ran when
+ # a documentation-only skip was expected, and a classifier replay
+ # that disagrees with the recorded classification. It also records
+ # the classification, the skipped work, runner minutes, and the
+ # required-check completion time in the job summary.
+ if: always()
+ needs:
+ - classify
+ - e2e-mock
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The gate replays the classifier against the same range.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Run validation gate
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ EVENT_NAME: ${{ github.event_name }}
+ PR_DRAFT: ${{ github.event.pull_request.draft }}
+ RUN_ID: ${{ github.run_id }}
+ HEAD_SHA: ${{ github.sha }}
+ FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+ WATCHED_JOBS: e2e-mock:always:gated
+ run: |
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml
index fd2a00747a..38aedaa8a1 100644
--- a/.github/workflows/mutation-testing.yml
+++ b/.github/workflows/mutation-testing.yml
@@ -2,7 +2,7 @@ name: Changed-code mutation testing
on:
pull_request:
- types: [edited, opened, reopened, ready_for_review, synchronize]
+ types: [edited, opened, reopened, ready_for_review, synchronize, labeled, unlabeled]
merge_group:
types: [checks_requested]
@@ -14,9 +14,43 @@ concurrency:
cancel-in-progress: true
jobs:
+ classify:
+ name: classify (mutation-testing)
+ # Classifies the pull request change range before any validation
+ # runs. Only an explicit documentation-only allowlist match can
+ # skip expensive jobs, and only for pull_request events; a failed
+ # or missing classification fails safe to full validation. The
+ # validation-gate job below audits this decision on every run.
+ runs-on: ubuntu-latest
+ outputs:
+ category: ${{ steps.classify.outputs.category }}
+ reason: ${{ steps.classify.outputs.reason }}
+ reason-code: ${{ steps.classify.outputs.reason-code }}
+ changed-files: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Classification resolves the merge base from history.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Classify changed files
+ id: classify
+ uses: ./.github/actions/classify-changes
+ with:
+ event-name: ${{ github.event_name }}
+ head-sha: ${{ github.sha }}
+ force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+
mutation-diff:
name: mutation-diff
- if: github.event_name == 'merge_group' || github.event.pull_request.draft == false
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue validation always runs.
+ if: ${{ !cancelled() && (github.event_name == 'merge_group' || github.event.pull_request.draft == false) && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -76,3 +110,43 @@ jobs:
echo "### Download mutation reports"
echo "[Open the changed-code-mutation-report artifact]($ARTIFACT_URL), then open the package's mutation.html file."
} >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary"
+
+ validation-gate:
+ name: validation-gate (mutation-testing)
+ # Always-running required gate for change-based validation. It
+ # rejects a missing or failed classification, validation jobs that
+ # did not run when full validation was required, jobs that ran when
+ # a documentation-only skip was expected, and a classifier replay
+ # that disagrees with the recorded classification. It also records
+ # the classification, the skipped work, runner minutes, and the
+ # required-check completion time in the job summary.
+ if: always()
+ needs:
+ - classify
+ - mutation-diff
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The gate replays the classifier against the same range.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Run validation gate
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ EVENT_NAME: ${{ github.event_name }}
+ PR_DRAFT: ${{ github.event.pull_request.draft }}
+ RUN_ID: ${{ github.run_id }}
+ HEAD_SHA: ${{ github.sha }}
+ FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+ WATCHED_JOBS: mutation-diff:pr-non-draft:gated
+ run: |
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml
index a3e7bb939b..744d55d162 100644
--- a/.github/workflows/visual-regression.yml
+++ b/.github/workflows/visual-regression.yml
@@ -7,7 +7,7 @@ on:
# that do not match the filter, which blocks merging for a required check.
# The merge_group event ignores path filters, so the queue is unaffected.
pull_request:
- types: [opened, reopened, ready_for_review, synchronize]
+ types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled]
merge_group:
types: [checks_requested]
@@ -19,7 +19,42 @@ permissions:
contents: read
jobs:
+ classify:
+ name: classify (visual-regression)
+ # Classifies the pull request change range before any validation
+ # runs. Only an explicit documentation-only allowlist match can
+ # skip expensive jobs, and only for pull_request events; a failed
+ # or missing classification fails safe to full validation. The
+ # validation-gate job below audits this decision on every run.
+ runs-on: ubuntu-latest
+ outputs:
+ category: ${{ steps.classify.outputs.category }}
+ reason: ${{ steps.classify.outputs.reason }}
+ reason-code: ${{ steps.classify.outputs.reason-code }}
+ changed-files: ${{ steps.classify.outputs.changed-files }}
+ allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # Classification resolves the merge base from history.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Classify changed files
+ id: classify
+ uses: ./.github/actions/classify-changes
+ with:
+ event-name: ${{ github.event_name }}
+ head-sha: ${{ github.sha }}
+ force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+
webview-visual:
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue validation always runs.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
timeout-minutes: 10
container:
@@ -55,6 +90,10 @@ jobs:
if-no-files-found: ignore
theme-fixtures:
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue validation always runs.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
@@ -84,6 +123,10 @@ jobs:
run: xvfb-run -a pnpm --filter @roo-code/vscode-e2e themes:check
extension-host-visual:
+ needs: classify
+ # Fail-safe: only a successful documentation-only classification on
+ # a pull request skips this job. Merge-queue validation always runs.
+ if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}
runs-on: ubuntu-latest
# Includes dependency setup, builds, repeated visuals, and production watchdog waits.
timeout-minutes: 35
@@ -124,3 +167,45 @@ jobs:
apps/vscode-e2e/playwright-report-electron
apps/vscode-e2e/test-results-electron
if-no-files-found: ignore
+
+ validation-gate:
+ name: validation-gate (visual-regression)
+ # Always-running required gate for change-based validation. It
+ # rejects a missing or failed classification, validation jobs that
+ # did not run when full validation was required, jobs that ran when
+ # a documentation-only skip was expected, and a classifier replay
+ # that disagrees with the recorded classification. It also records
+ # the classification, the skipped work, runner minutes, and the
+ # required-check completion time in the job summary.
+ if: always()
+ needs:
+ - classify
+ - webview-visual
+ - theme-fixtures
+ - extension-host-visual
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The gate replays the classifier against the same range.
+ fetch-depth: 0
+ # This job never pushes — don't persist the GITHUB_TOKEN.
+ persist-credentials: false
+ - name: Run validation gate
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ NEEDS_JSON: ${{ toJSON(needs) }}
+ EVENT_NAME: ${{ github.event_name }}
+ PR_DRAFT: ${{ github.event.pull_request.draft }}
+ RUN_ID: ${{ github.run_id }}
+ HEAD_SHA: ${{ github.sha }}
+ FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }}
+ FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }}
+ WATCHED_JOBS: webview-visual:always:gated,theme-fixtures:always:gated,extension-host-visual:always:gated
+ run: |
+ BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")"
+ node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 61bb417bf2..38e7032c02 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -25,6 +25,8 @@ Zoo Code is a community project. We value every contribution. We work on an [Iss
- [Writing Style](#writing-style)
- [Legal](#legal)
+See [change-based CI validation](docs/ci-change-based-validation.md) for validation skips and full-validation overrides.
+
## Before You Contribute
### 1. Code of Conduct
diff --git a/docs/ci-change-based-validation.md b/docs/ci-change-based-validation.md
new file mode 100644
index 0000000000..3a2763752f
--- /dev/null
+++ b/docs/ci-change-based-validation.md
@@ -0,0 +1,226 @@
+# Change-based validation with an audit gate
+
+Every pull request update previously ran the full validation suite even when the
+change could not affect product behavior. This document describes the
+conservative change-based validation introduced for issue #2291, its safety
+model, its override and rollback paths, and the evidence collected before
+skips were enabled.
+
+## How it works
+
+Five workflows (`code-qa`, `e2e`, `visual-regression`, `mutation-testing`,
+`codeql`) run a fast `classify` job first. It runs
+`scripts/change-classification.mjs` over the pull request's full change range
+and emits either `docs-only` or `full`. Expensive jobs read that result through
+their `needs` graph:
+
+- A job is skipped only when classification succeeded, the event is
+ `pull_request`, and the category is exactly `docs-only`. The job-level
+ condition reports `skipped`, and GitHub treats a skipped required check as
+ successful. The required unit-test matrix jobs start both platform checks
+ and gate their steps instead, because GitHub evaluates job conditions before
+ expanding a matrix. Skipping the matrix job itself may leave its named
+ required contexts unreported.
+ The required `compile` job also starts every time and fails unless the
+ always-running formatting and validation-policy tests passed; only its
+ expensive steps are gated. This preserves the existing check names without a ruleset change; live
+ verification is still pending.
+- Every other outcome runs the job unchanged: category `full`, a failed
+ classification, a missing classification, or any non-`pull_request` event
+ (merge queue, pushes, scheduled runs).
+
+Each workflow also runs a `validation-gate` job that always executes.
+The classify and gate checks have unique display names, such as
+`classify (code-qa)` and `validation-gate (code-qa)`, for branch rulesets. The gate
+replays the classifier against the same commit range and rejects the run when:
+
+- the classification is missing, malformed, or non-terminal,
+- a validation job did not run when full validation was required,
+- a validation job ran when a documentation-only skip was expected,
+- the replay disagrees with the recorded classification,
+- a watched job is missing from the `needs` graph.
+
+On merge-queue events classification never reduces validation; existing
+event and draft conditions still apply, including skips for PR-only jobs.
+CodeQL does not currently subscribe to merge-queue events. The gate writes a job
+summary with the classification, the expected and actual outcome of every
+watched job, best-effort runner minutes per job, the total runner minutes of the run, and
+the time from run start to the last completed job (the required-check
+completion time).
+
+## The documentation-only allowlist
+
+`scripts/change-classification.mjs` holds one explicit allowlist. Only these
+files can produce a `docs-only` classification:
+
+| Pattern | Reason |
+| -------------------- | ------------------------------------------------- |
+| `CONTRIBUTING.md` | Contributor guide. |
+| `SECURITY.md` | Security policy. |
+| `PRIVACY.md` | Privacy notice. |
+| `CODE_OF_CONDUCT.md` | Code of conduct. |
+| `docs/**/*.md` | Documentation tree; contains only Markdown today. |
+| `.github/**/*.md` | Issue and pull request templates. |
+
+The allowlist is deliberately narrow:
+
+- Release inputs stay out. `CHANGELOG.md`, `src/CHANGELOG.md`, `locales/*/README.md`,
+ `README.md` (copied by `src/esbuild.mjs` into the packaged extension),
+ and packaging manifests trigger full validation, and the separate
+ release-validation workflow is untouched.
+- Workflow and toolchain configuration stays out. Only `.md` files under
+ `.github/` match; executable workflow/action YAML and `CODEOWNERS` select
+ full validation. Inert Markdown notes under workflows or actions may skip.
+- Embedded fixtures, executable examples, and generated inputs never skip:
+ any path containing `fixtures`, `__tests__`, `__mocks__`, `test-utils`,
+ `snapshots`, `__snapshots__`, `generated`, `examples`, or `testdata` is
+ rejected even under an allowlisted prefix.
+- The `format-check` job always runs because `pnpm format:check` validates
+ every Markdown file; documentation changes keep their own check.
+- The `knip` and `invisible-chars` jobs always run. `dependency-review` is
+ never gated by classification but stays pull-request-only. Knip is
+ deliberately unconditional because a repository invariant test requires
+ the knip job to run without conditions or allowed failure.
+
+## Documentation input audit
+
+The audit at `687f8489d` checked references to the allowlisted paths and their
+file names across tracked source, tests, build scripts, and CI configuration.
+It also checked package build entry points, test discovery, asset copying,
+Turbo inputs, and VSIX packaging rules.
+
+| Surface | Evidence | Result |
+| -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Extension bundling | `src/esbuild.mjs` calls `copyPaths()` with an explicit source list. `packages/build/src/esbuild.ts` copies only those configured paths. | Root `README.md` and `CHANGELOG.md` are package inputs and remain outside the allowlist. No allowlisted documentation is copied. |
+| VSIX packaging | `src/.vscodeignore` excludes everything, then includes the built extension, webview, assets, and release metadata. | Root `docs/`, `.github/`, and the four allowlisted root files are not packaged. |
+| Webview and package builds | `webview-ui/vite.config.ts`, `apps/cli/tsup.config.ts`, and `packages/types/tsup.config.ts` select code entry points and assets. | No imports, raw-file loads, or asset rules select allowlisted documentation. |
+| Tests and models | Vitest configurations select code tests. Searches covered `docs/`, current documentation basenames, Markdown imports, and filesystem readers. | Documentation references are comments, temporary repository files, mocked paths, or workspace examples. Tests do not read the checked-in allowlisted documents. |
+| Turbo and Knip | Root and package Turbo files define package-local inputs. `knip.json` selects code entry points and projects. | No explicit input reaches the root documentation tree. Default package inputs can include package-local Markdown, which remains outside the allowlist. |
+| Release validation | `.github/workflows/release-validation.yml` builds the extension and compares root `README.md` with the copied package README. | Release inputs retain full validation. The release workflow is unchanged. |
+
+Generic runtime tools can read Markdown in a user's workspace. This does not
+make repository documentation a build or test input. The audit found no reason
+to narrow the current allowlist further. This is source-level evidence for the
+current tree, not a guarantee about future consumers. A new documentation
+reader must update the classification policy and its regression tests.
+
+## Fail-safe behavior
+
+| Input | Classification |
+| --------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- |
+| All changed files match the allowlist (additions, modifications, deletions, documentation-to-documentation renames, type changes) | `docs-only` |
+| Any path outside the allowlist, including source, dependency manifests, lockfiles, workflow files | `full` |
+| Rename from or into a non-allowlisted path (both sides are classified) | `full` |
+| Empty change range | `full` |
+| Change status the classifier does not recognize (for example copies) | `full` |
+| Missing or unresolvable merge base, shallow history, unknown commits | Classification fails; jobs fall back to full validation; the gate records the fallback |
+| Any git error or malformed diff output | Classification fails; jobs fall back to full validation |
+
+The changed-file enumeration uses NUL-delimited `git diff --name-status` over
+the full merge-base range with rename detection, so it cannot silently
+truncate on whitespace or large diffs. Rename entries classify the old and the
+new path.
+
+Changes to `scripts/change-classification.mjs` or `scripts/change-gate.mjs`
+always select full validation, independently of the documentation allowlist.
+The gate also checks these paths directly with git against the base range
+before accepting a documentation-only result. An allowlist expansion cannot
+classify its own policy change as documentation-only. The fingerprint remains
+an observability value rather than a trust boundary.
+
+## Override and rollback
+
+Full validation can be forced at three levels:
+
+1. **Per pull request:** add the `ci:full-validation` label. The classify job
+ reports `full` with reason `override-label`.
+2. **Repository-wide:** set the repository variable
+ `ZOO_CODE_REQUIRE_FULL_VALIDATION` to `true`. No code change is needed;
+ every classification reports `full` with reason `override-variable`.
+3. **Permanent:** revert the pull request that introduced this change. All
+ workflows then run exactly as before.
+
+The label must exist in the repository before it can be applied. Adding or
+removing a label starts a fresh run in all five workflows. Their concurrency
+groups replace superseded PR runs, so the latest label state determines the
+active classification. Repository variables do not trigger runs: push an update
+or dispatch a fresh run after changing the variable. Re-running an old run
+retains its original event payload.
+
+The gate applies the same override rules when it replays the classifier, so an
+overridden run still validates coherence.
+
+## Historical validation before enabling skips
+
+The original pre-enablement review snapshot (PR #2325 at `d13547de0`)
+reported the following results over the 500 most recent
+first-parent commits of `main` with `pull_request` semantics
+(`node scripts/change-classification.mjs history --limit 500`). These are
+historical results, not a current measurement; the original capture date was
+not recorded. To reproduce the sample, fetch
+complete history first (`git fetch --unshallow` for a shallow checkout), check
+out the snapshot, and record the date and main SHA with the report:
+
+- 499 commits classified `full` (reason `non-documentation-path`).
+- 1 commit classified `docs-only`: `a704002b67fd`, "[Docs] Specify P1
+ persisted ownership and add the run-state kernel target architecture
+ (#1706)". Its eight files are all `docs/architecture/*.md`; skipping
+ comprehensive validation for that change is correct.
+- 0 classification failures and 0 unrecognized change statuses: the parser
+ handled every real change set in the sample, including renames and
+ deletions.
+
+The sample starts narrow by design: about 0.2% of changes would have skipped,
+and the only proposed skip was verified file by file. Skipped work is always
+explained in the gate summary.
+
+## Expanding the allowlist
+
+Expand the allowlist only with dependency evidence:
+
+1. Prove the new pattern is not a build, runtime, or test input: grep the
+ test suites, build scripts, Knip configuration, and Turbo inputs for reads
+ of the path.
+2. Extend `scripts/change-classification.test.mjs` with accept and reject
+ cases for the new pattern.
+3. Run the history mode over a large sample and verify every newly proposed
+ skip by hand.
+4. Expand one pattern at a time and watch the gate summaries for unexpected
+ skips.
+
+Broader package selection (running only affected packages) remains out of
+scope until the shared-input audit is done; Zoo imports shared source outside
+package-local directories. Existing coverage slicing (issue #47) is a separate
+effort. A comprehensive merge-queue run is additional protection, not a
+substitute for correct pull request classification.
+
+## Trust boundary and maintenance
+
+Replay detects accidental disagreement. On `pull_request`, workflow, action,
+classifier, and gate code all come from the PR merge commit; the gate is not
+independent protection against a PR author who edits validation itself.
+Maintainers should review `.github/**` and `scripts/change-*.mjs` as CI policy;
+CODEOWNERS protection for these paths is a possible separate policy change.
+The gate jobs must be added to branch rulesets if maintainers want gate failures
+themselves to block merging. Formatting and policy-test failures already flow
+through the existing required `compile` context.
+
+The repeated gate jobs remain explicit to keep workflow needs and check names
+visible. Extracting a shared gate action is deferred to a separate refactor.
+
+## Current verification and open work
+
+No live docs-only PR run has confirmed the two platform check contexts yet.
+Local workflow tests verify their names and wiring; this does not establish
+GitHub's live check-reporting behavior.
+
+`format-check` and all `validation-gate` jobs are not required checks in the
+current ruleset. Formatting and policy-test failures block via required
+`compile`. Making audit-gate failures block directly still requires a separate
+ruleset update. The gate reads job results and cannot observe step-level gating
+inside `compile` or `unit-test`; workflow tests pin those step conditions.
+
+All label changes currently start all five workflows and may cancel in-flight
+PR validation, including mutation testing. This runner-time cost is accepted
+for now to keep override event handling simple and consistent. A shared gate
+action and broader Stryker integration remain follow-ups.
diff --git a/package.json b/package.json
index a67f824dcc..a8bf97f0e9 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
"test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts",
"agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json",
"test:mutation-ci": "node --test scripts/stryker-diff.test.mjs",
+ "test:change-gate": "node --test scripts/change-classification.test.mjs scripts/change-gate.test.mjs",
"lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-file-replacement-restart.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check",
"webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts",
"webview-recovery:check-types": "tsc -p scripts/tsconfig.webview-recovery.json",
diff --git a/scripts/change-classification.mjs b/scripts/change-classification.mjs
new file mode 100644
index 0000000000..40b18854f3
--- /dev/null
+++ b/scripts/change-classification.mjs
@@ -0,0 +1,440 @@
+#!/usr/bin/env node
+
+// Conservative change-based validation classifier (issue #2291).
+//
+// Classifies the full changed-file range of a pull request as
+// "docs-only" (safe to skip expensive comprehensive validation) or
+// "full" (run everything). The classification is fail-safe: every
+// uncertain input — unknown paths, unrecognized change statuses,
+// missing history, malformed git output, empty ranges — selects full
+// validation. Only an explicit allowlist of documentation-only files
+// whose contents are not build, runtime, or test inputs can produce
+// "docs-only", and only for pull_request events.
+
+import { execFileSync } from "node:child_process"
+import { createHash } from "node:crypto"
+import fs from "node:fs"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+
+// Exact repository files that are pure documentation. Enumerated
+// individually so an unexpected new root-level markdown file (for
+// example a release-affecting CHANGELOG.md) never matches by accident.
+const DOCS_ONLY_FILES = [
+ { path: "CONTRIBUTING.md", description: "contributor guide" },
+ { path: "SECURITY.md", description: "security policy" },
+ { path: "PRIVACY.md", description: "privacy notice" },
+ { path: "CODE_OF_CONDUCT.md", description: "code of conduct" },
+]
+
+// Directories where only Markdown files qualify. Everything else under
+// these prefixes (images, configs, scripts) still selects full
+// validation.
+const DOCS_ONLY_MARKDOWN_PREFIXES = [
+ { prefix: "docs/", description: "documentation tree" },
+ { prefix: ".github/", description: "repository templates" },
+]
+
+// Defense in depth: these path segments mark files that may be loaded
+// by builds, tests, or examples even when they sit under an allowlisted
+// prefix. A Markdown file inside a fixtures or examples directory is a
+// build/runtime/test input, so it must never allow a skip.
+export const NEVER_SKIP_SEGMENTS = [
+ "fixtures",
+ "__tests__",
+ "__mocks__",
+ "test-utils",
+ "snapshots",
+ "__snapshots__",
+ "generated",
+ "examples",
+ "testdata",
+]
+
+// Policy paths stay protected even if the documentation allowlist expands.
+const VALIDATION_POLICY_PATHS = new Set(["scripts/change-classification.mjs", "scripts/change-gate.mjs"])
+
+// Statuses understood from `git diff --name-status`. Any other letter
+// (including C for copies and X for unknown) is unrecognized and fails
+// safe to full validation.
+const KNOWN_STATUSES = new Set(["A", "M", "D", "R", "T"])
+
+export const SKIP_ELIGIBLE_EVENT = "pull_request"
+
+export const OUTPUT_REASONS = {
+ "non-pr-event": "Only pull_request events may skip; merge queue, push, and scheduled runs keep full validation.",
+ "empty-change-range": "The change range contained no files; an empty enumeration is treated as uncertain.",
+ "unrecognized-status": "The diff contained a change status the classifier does not understand.",
+ "validation-policy-change": "The classifier or validation gate changed; full validation is required.",
+ "non-documentation-path": "The change range contained paths outside the documentation-only allowlist.",
+ "protected-path-segment":
+ "An allowlisted path contained a protected segment (fixtures, tests, generated, examples).",
+ "override-label": "Full validation was forced by the ci:full-validation pull request label.",
+ "override-variable": "Full validation was forced by the ZOO_CODE_REQUIRE_FULL_VALIDATION repository variable.",
+}
+
+export function allowlistFingerprint() {
+ return createHash("sha256")
+ .update(
+ JSON.stringify({
+ files: DOCS_ONLY_FILES,
+ prefixes: DOCS_ONLY_MARKDOWN_PREFIXES,
+ never: NEVER_SKIP_SEGMENTS,
+ }),
+ )
+ .digest("hex")
+ .slice(0, 12)
+}
+
+function hasProtectedSegment(filePath) {
+ return NEVER_SKIP_SEGMENTS.some((segment) => filePath.split("/").includes(segment))
+}
+
+export function isDocsOnlyPath(filePath) {
+ if (typeof filePath !== "string" || filePath.length === 0) return false
+ const normalized = path.posix.normalize(filePath)
+ if (normalized.startsWith("../") || path.posix.isAbsolute(normalized)) return false
+ if (hasProtectedSegment(normalized)) return false
+ if (DOCS_ONLY_FILES.some((entry) => entry.path === normalized)) return true
+ if (!normalized.endsWith(".md")) return false
+ return DOCS_ONLY_MARKDOWN_PREFIXES.some((entry) => normalized.startsWith(entry.prefix))
+}
+
+// Parses `git diff --name-status -z --find-renames` output. Mirrors the
+// parser in scripts/stryker-diff.mjs (rename and copy entries carry the
+// old and the new path; all other statuses carry one path) but stays
+// self-contained so classification can run without installed
+// dependencies.
+export function parseNameStatus(output) {
+ const tokens = output.split("\0")
+ const entries = []
+
+ for (let index = 0; index < tokens.length;) {
+ const statusToken = tokens[index++]
+ if (!statusToken) break
+
+ const status = statusToken[0]
+ if (status === "R" || status === "C") {
+ const oldPath = tokens[index++]
+ const newPath = tokens[index++]
+ if (!oldPath || !newPath) throw new Error(`Malformed git name-status entry: ${statusToken}`)
+ entries.push({ status, oldPath, path: newPath })
+ } else {
+ const filePath = tokens[index++]
+ if (!filePath) throw new Error(`Malformed git name-status entry: ${statusToken}`)
+ entries.push({ status, path: filePath })
+ }
+ }
+
+ return entries
+}
+
+function underAllowlistedPrefix(filePath) {
+ return DOCS_ONLY_MARKDOWN_PREFIXES.some((entry) => filePath.startsWith(entry.prefix))
+}
+
+// Pure classification over an already-parsed change list. Unknown
+// statuses, empty ranges, and any non-allowlisted path fail safe to
+// full validation.
+export function classifyEntries(entries) {
+ if (entries.length === 0) {
+ return { category: "full", reasonCode: "empty-change-range", changedFiles: 0, docsFiles: [], otherFiles: [] }
+ }
+
+ if (entries.some((entry) => !KNOWN_STATUSES.has(entry.status[0]))) {
+ return {
+ category: "full",
+ reasonCode: "unrecognized-status",
+ changedFiles: entries.length,
+ docsFiles: [],
+ otherFiles: entries.map(({ path: filePath }) => filePath),
+ }
+ }
+
+ // Policy changes cannot grant themselves a skip by widening the allowlist.
+ const policyPaths = entries
+ .flatMap((entry) => [entry.path, entry.oldPath])
+ .filter((filePath) => VALIDATION_POLICY_PATHS.has(filePath))
+ if (policyPaths.length > 0) {
+ return {
+ category: "full",
+ reasonCode: "validation-policy-change",
+ changedFiles: entries.length,
+ docsFiles: [],
+ offendingPaths: policyPaths,
+ }
+ }
+
+ const docsFiles = []
+ const otherFiles = []
+
+ for (const entry of entries) {
+ const paths = [entry.path, ...(entry.oldPath ? [entry.oldPath] : [])]
+ for (const filePath of paths) {
+ const target = isDocsOnlyPath(filePath) ? docsFiles : otherFiles
+ target.push({ status: entry.status, path: filePath })
+ }
+ }
+
+ if (otherFiles.length > 0) {
+ const offendingPaths = otherFiles.map(({ path: filePath }) => filePath)
+ const allProtected = offendingPaths.every(
+ (filePath) => hasProtectedSegment(filePath) && underAllowlistedPrefix(filePath),
+ )
+ return {
+ category: "full",
+ reasonCode: allProtected ? "protected-path-segment" : "non-documentation-path",
+ changedFiles: entries.length,
+ docsFiles,
+ otherFiles,
+ offendingPaths: offendingPaths.slice(0, 5),
+ }
+ }
+
+ return { category: "docs-only", reasonCode: "allowlisted-documentation", changedFiles: entries.length, docsFiles }
+}
+
+// Classification result for a whole event. Non-pull_request events are
+// never skip-eligible; the range is still computed for observability.
+export function classifyEvent({ eventName, entries, forceFullReasonCode }) {
+ const result = classifyEntries(entries)
+ const skipEligible = eventName === SKIP_ELIGIBLE_EVENT
+
+ if (!skipEligible) {
+ return { ...result, category: "full", reasonCode: "non-pr-event" }
+ }
+ if (forceFullReasonCode && OUTPUT_REASONS[forceFullReasonCode]) {
+ return { ...result, category: "full", reasonCode: forceFullReasonCode }
+ }
+ return result
+}
+
+function git(repoRoot, args) {
+ return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8", maxBuffer: 20 * 1024 * 1024 })
+}
+
+export function resolveCommits(repoRoot, baseSha, headSha) {
+ if (!/^[0-9a-f]{40}$/i.test(baseSha) || !/^[0-9a-f]{40}$/i.test(headSha)) {
+ throw new Error("Base and head must be full 40-character commit SHAs")
+ }
+ for (const [label, sha] of [
+ ["base", baseSha],
+ ["head", headSha],
+ ]) {
+ try {
+ git(repoRoot, ["cat-file", "-e", `${sha}^{commit}`])
+ } catch (error) {
+ throw new Error(`Could not resolve ${label} commit ${sha} in the checkout (shallow or missing history)`)
+ }
+ }
+}
+
+export function changedEntries(repoRoot, baseSha, headSha) {
+ resolveCommits(repoRoot, baseSha, headSha)
+ let mergeBase
+ try {
+ mergeBase = git(repoRoot, ["merge-base", baseSha, headSha]).trim()
+ git(repoRoot, ["merge-base", "--is-ancestor", mergeBase, headSha])
+ } catch (error) {
+ throw new Error("Could not resolve a common merge base between the base and head commits")
+ }
+ const nameStatus = git(repoRoot, ["diff", "--name-status", "-z", "--find-renames", `${mergeBase}...${headSha}`])
+ const entries = parseNameStatus(nameStatus)
+ return { mergeBase, entries }
+}
+
+function writeOutput(name, value) {
+ if (!process.env.GITHUB_OUTPUT) return
+ fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`)
+}
+
+function escapeSummaryCell(value) {
+ return String(value).replaceAll("|", "\\|").replaceAll("\n", " ")
+}
+
+function escapeSummaryPath(value) {
+ const characters = Array.from(String(value))
+ // Encode punctuation before Markdown parsing; cap individual paths in summaries.
+ const text = characters.slice(0, 500).join("")
+ return (
+ text.replace(/[^\p{L}\p{N} /-]/gu, (character) => `${character.codePointAt(0)};`) +
+ (characters.length > 500 ? "…" : "")
+ )
+}
+
+export function formatSummary(result, { eventName, mergeBase, headSha } = {}) {
+ const lines = [
+ "## Change-based validation classification",
+ "",
+ "| Field | Value |",
+ "| --- | --- |",
+ `| Event | ${escapeSummaryCell(eventName ?? "unknown")} |`,
+ `| Category | ${escapeSummaryCell(result.category)} |`,
+ `| Reason | ${escapeSummaryCell(OUTPUT_REASONS[result.reasonCode] ?? result.reasonCode)} |`,
+ `| Changed files | ${result.changedFiles} |`,
+ `| Allowlist fingerprint | ${allowlistFingerprint()} |`,
+ ]
+ if (headSha) lines.push(`| Head | ${headSha.slice(0, 12)} |`)
+ if (mergeBase) lines.push(`| Merge base | ${mergeBase.slice(0, 12)} |`)
+ lines.push("")
+
+ if (result.category === "docs-only") {
+ lines.push(
+ "Every changed file matches the documentation-only allowlist. Comprehensive PR validation jobs are skipped; the validation gate verifies this decision.",
+ "",
+ "Allowlisted files
",
+ "",
+ )
+ for (const { path: filePath } of result.docsFiles) lines.push(`- ${escapeSummaryPath(filePath)}`)
+ lines.push("", " ")
+ } else if (result.reasonCode === "non-pr-event") {
+ lines.push(
+ "Full validation runs unchanged for this event. Merge-queue, release, and push validation is never reduced by classification.",
+ )
+ } else {
+ lines.push("Full validation runs because the change range is not proven documentation-only.")
+ const details = result.offendingPaths ?? result.docsFiles.map(({ path: filePath }) => filePath)
+ if (details.length > 0) {
+ lines.push("", "Paths outside or straddling the allowlist
", "")
+ for (const filePath of details) lines.push(`- ${escapeSummaryPath(filePath)}`)
+ lines.push("", " ")
+ }
+ }
+ return `${lines.join("\n")}\n`
+}
+
+function argument(name) {
+ const index = process.argv.indexOf(name)
+ return index === -1 ? undefined : process.argv[index + 1]
+}
+
+function repoRootFromArgv() {
+ return path.resolve(argument("--repo-root") ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."))
+}
+
+function emitResult(result, context) {
+ writeOutput("category", result.category)
+ writeOutput("reason-code", result.reasonCode)
+ writeOutput("reason", OUTPUT_REASONS[result.reasonCode] ?? result.reasonCode)
+ writeOutput("changed-files", String(result.changedFiles))
+ writeOutput("allowlist-fingerprint", allowlistFingerprint())
+ console.log(
+ `Change classification: ${result.category} (${result.reasonCode}), ${result.changedFiles} changed file(s)`,
+ )
+ if (process.env.GITHUB_STEP_SUMMARY) {
+ fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(result, context))
+ }
+}
+
+// Walks first-parent history and classifies each commit as if it were a
+// pull request change range. Used to validate the allowlist against
+// representative historical changes before skips are trusted.
+export function runHistory(repoRoot, { limit = 200 } = {}) {
+ const log = git(repoRoot, ["log", "--first-parent", `--max-count=${limit}`, "--format=%H%x09%s"])
+ const commits = log
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => {
+ const [sha, subject] = line.split("\t")
+ const prMatch = /\(#(\d+)\)$/.exec(subject) ?? /Merge pull request #(\d+) from/.exec(subject)
+ return { sha, subject, pr: prMatch ? Number(prMatch[1]) : undefined }
+ })
+
+ const rows = []
+ for (const commit of commits) {
+ const parents = git(repoRoot, ["rev-list", "--parents", "-n", "1", commit.sha]).trim().split(/\s+/)
+ const baseSha = parents[1]
+ if (!baseSha) continue
+ try {
+ const { entries } = changedEntries(repoRoot, baseSha, commit.sha)
+ const result = classifyEvent({ eventName: SKIP_ELIGIBLE_EVENT, entries })
+ rows.push({ ...commit, baseSha, ...result })
+ } catch (error) {
+ rows.push({
+ ...commit,
+ baseSha,
+ category: "classifier-failure",
+ reasonCode: error.message,
+ changedFiles: 0,
+ })
+ }
+ }
+ return rows
+}
+
+function formatHistory(rows) {
+ const skipCount = rows.filter((row) => row.category === "docs-only").length
+ const lines = [
+ `Classified ${rows.length} first-parent commit(s); ${skipCount} would skip comprehensive validation.`,
+ "",
+ "| Commit | PR | Category | Reason | Files | Proposed skip file list |",
+ "| --- | --- | --- | --- | ---: | --- |",
+ ]
+ for (const row of rows) {
+ const skipList =
+ row.category === "docs-only"
+ ? (row.docsFiles ?? [])
+ .map(({ path: filePath }) => escapeSummaryPath(filePath))
+ .join(", ")
+ .slice(0, 180)
+ : ""
+ lines.push(
+ `| ${row.sha.slice(0, 12)} | ${row.pr ?? "—"} | ${row.category} | ${escapeSummaryCell(row.reasonCode)} | ${row.changedFiles} | ${skipList} |`,
+ )
+ }
+ return `${lines.join("\n")}\n`
+}
+
+function main() {
+ const repoRoot = repoRootFromArgv()
+ const command = process.argv[2]
+
+ if (command === "ci") {
+ const eventName = argument("--event")
+ const headSha = argument("--head")
+ if (!eventName || !headSha)
+ throw new Error(
+ "Usage: change-classification.mjs ci --event --head [--repo-root ] [--force-full-label] [--force-full-variable]",
+ )
+ const baseSha = argument("--base") ?? git(repoRoot, ["rev-parse", `${headSha}^1`]).trim()
+ const forceFullReasonCode = process.argv.includes("--force-full-label")
+ ? "override-label"
+ : process.argv.includes("--force-full-variable")
+ ? "override-variable"
+ : undefined
+ const { mergeBase, entries } = changedEntries(repoRoot, baseSha, headSha)
+ const result = classifyEvent({ eventName, entries, forceFullReasonCode })
+ emitResult(result, { eventName, mergeBase, headSha })
+ return
+ }
+
+ if (command === "history") {
+ const limit = Number(argument("--limit") ?? 200)
+ const rows = runHistory(repoRoot, { limit })
+ const output = argument("--output")
+ if (output) {
+ fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true })
+ fs.writeFileSync(path.resolve(output), formatHistory(rows))
+ const jsonPath = /\.md$/.test(output)
+ ? path.resolve(output).replace(/\.md$/, ".json")
+ : `${path.resolve(output)}.json`
+ fs.writeFileSync(jsonPath, JSON.stringify(rows, null, 2))
+ console.log(`History classification written to ${output}`)
+ } else {
+ process.stdout.write(formatHistory(rows))
+ }
+ return
+ }
+
+ throw new Error("Usage: change-classification.mjs [options]")
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ try {
+ main()
+ } catch (error) {
+ console.error(`::error title=Change classification::${String(error.message).replaceAll("\n", " ")}`)
+ console.error(`Change classification failed: ${error.message}`)
+ process.exitCode = 1
+ }
+}
diff --git a/scripts/change-classification.test.mjs b/scripts/change-classification.test.mjs
new file mode 100644
index 0000000000..eab294bba5
--- /dev/null
+++ b/scripts/change-classification.test.mjs
@@ -0,0 +1,721 @@
+import assert from "node:assert/strict"
+import { execFileSync, spawnSync } from "node:child_process"
+import fs from "node:fs"
+import os from "node:os"
+import path from "node:path"
+import { describe, it } from "node:test"
+import { fileURLToPath } from "node:url"
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
+const classifierPath = path.join(repositoryRoot, "scripts/change-classification.mjs")
+
+import { replayClassification, validationPolicyChanged } from "./change-gate.mjs"
+
+import {
+ NEVER_SKIP_SEGMENTS,
+ resolveCommits,
+ allowlistFingerprint,
+ changedEntries,
+ classifyEntries,
+ classifyEvent,
+ formatSummary,
+ isDocsOnlyPath,
+ parseNameStatus,
+ runHistory,
+} from "./change-classification.mjs"
+
+describe("documentation-only allowlist", () => {
+ it("accepts the enumerated root documentation files", () => {
+ for (const filePath of ["CONTRIBUTING.md", "SECURITY.md", "PRIVACY.md", "CODE_OF_CONDUCT.md"]) {
+ assert.ok(isDocsOnlyPath(filePath), `${filePath} should be allowlisted`)
+ }
+ })
+
+ it("accepts markdown under the documentation and template prefixes", () => {
+ for (const filePath of [
+ "docs/development-quality-checks.md",
+ "docs/architecture/task-lifecycle-model.md",
+ ".github/pull_request_template.md",
+ ".github/ISSUE_TEMPLATE/bug_report.md",
+ ]) {
+ assert.ok(isDocsOnlyPath(filePath), `${filePath} should be allowlisted`)
+ }
+ })
+
+ it("rejects files that are build, runtime, test, or release inputs", () => {
+ for (const filePath of [
+ "README.md",
+ "CHANGELOG.md",
+ "src/CHANGELOG.md",
+ "AGENTS.md",
+ "src/README.md",
+ "locales/en/README.md",
+ "package.json",
+ "pnpm-lock.yaml",
+ "docs/image.png",
+ "docs/scripts/example.js",
+ ".github/workflows/code-qa.yml",
+ ".github/CODEOWNERS",
+ ".github/actions/classify-changes/action.yml",
+ "src/core/task/index.ts",
+ ]) {
+ assert.equal(isDocsOnlyPath(filePath), false, `${filePath} must not be allowlisted`)
+ }
+ })
+
+ it("never skips protected path segments even under allowlisted prefixes", () => {
+ for (const filePath of [
+ "docs/fixtures/sample.md",
+ "docs/__tests__/case.md",
+ "docs/generated/report.md",
+ "docs/examples/walkthrough.md",
+ "docs/test-utils/guide.md",
+ ".github/ISSUE_TEMPLATE/fixtures/template.md",
+ ]) {
+ assert.equal(isDocsOnlyPath(filePath), false, `${filePath} must not be allowlisted`)
+ }
+ })
+
+ it("rejects traversal, absolute paths, and empty values", () => {
+ for (const filePath of ["../outside.md", "/etc/passwd.md", "", undefined, null]) {
+ assert.equal(isDocsOnlyPath(filePath), false)
+ }
+ })
+})
+
+for (const segment of NEVER_SKIP_SEGMENTS) {
+ it(`protects segment ${segment}`, () => {
+ for (const prefix of ["docs", ".github"]) assert.equal(isDocsOnlyPath(`${prefix}/${segment}/case.md`), false)
+ })
+}
+
+describe("name-status parsing", () => {
+ it("parses single-path entries", () => {
+ assert.deepEqual(parseNameStatus("A\0docs/new.md\0M\0CONTRIBUTING.md\0D\0src/gone.ts\0"), [
+ { status: "A", path: "docs/new.md" },
+ { status: "M", path: "CONTRIBUTING.md" },
+ { status: "D", path: "src/gone.ts" },
+ ])
+ })
+
+ it("parses rename entries with old and new paths", () => {
+ assert.deepEqual(parseNameStatus("R100\0docs/old.md\0docs/new.md\0M\0src/a.ts\0"), [
+ { status: "R", oldPath: "docs/old.md", path: "docs/new.md" },
+ { status: "M", path: "src/a.ts" },
+ ])
+ })
+
+ it("rejects malformed entries", () => {
+ assert.throws(() => parseNameStatus("R100\0only-one-path\0"))
+ assert.throws(() => parseNameStatus("M\0"))
+ })
+})
+
+describe("change classification", () => {
+ it("fails safe on an empty change range", () => {
+ const result = classifyEntries([])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "empty-change-range")
+ })
+
+ it("classifies an all-documentation range including deletions and renames", () => {
+ const result = classifyEntries([
+ { status: "A", path: "docs/new.md" },
+ { status: "M", path: "CONTRIBUTING.md" },
+ { status: "D", path: "docs/gone.md" },
+ { status: "R", oldPath: "docs/old.md", path: "docs/renamed.md" },
+ { status: "T", path: "docs/symlinked.md" },
+ ])
+ assert.equal(result.category, "docs-only")
+ assert.equal(result.changedFiles, 5)
+ })
+
+ it("fails safe when a rename moves documentation into source", () => {
+ const result = classifyEntries([{ status: "R", oldPath: "docs/old.md", path: "src/moved.md" }])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "non-documentation-path")
+ assert.ok(result.offendingPaths.includes("src/moved.md"))
+ })
+
+ it("fails safe when a rename moves source into documentation", () => {
+ const result = classifyEntries([{ status: "R", oldPath: "src/moved.ts", path: "docs/moved.md" }])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "non-documentation-path")
+ assert.ok(result.offendingPaths.includes("src/moved.ts"))
+ })
+
+ it("fails safe on mixed documentation and source changes", () => {
+ const result = classifyEntries([
+ { status: "M", path: "CONTRIBUTING.md" },
+ { status: "M", path: "src/core/task/index.ts" },
+ ])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "non-documentation-path")
+ })
+
+ it("fails safe on protected segments inside the documentation tree", () => {
+ const result = classifyEntries([{ status: "R", oldPath: "docs/a.md", path: "docs/fixtures/b.md" }])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "protected-path-segment")
+ })
+
+ it("fails safe on unrecognized change statuses", () => {
+ const result = classifyEntries([{ status: "C", oldPath: "docs/a.md", path: "docs/b.md" }])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "unrecognized-status")
+ })
+})
+
+describe("event gating", () => {
+ const docsEntries = [{ status: "M", path: "docs/only.md" }]
+
+ it("allows a skip only for documentation-only pull request ranges", () => {
+ assert.equal(classifyEvent({ eventName: "pull_request", entries: docsEntries }).category, "docs-only")
+ })
+
+ it("forces full validation for merge queue, push, and scheduled events", () => {
+ for (const eventName of ["merge_group", "push", "schedule", "workflow_dispatch"]) {
+ const result = classifyEvent({ eventName, entries: docsEntries })
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "non-pr-event")
+ }
+ })
+
+ it("honors explicit full-validation overrides", () => {
+ const label = classifyEvent({
+ eventName: "pull_request",
+ entries: docsEntries,
+ forceFullReasonCode: "override-label",
+ })
+ assert.equal(label.category, "full")
+ assert.equal(label.reasonCode, "override-label")
+
+ const variable = classifyEvent({
+ eventName: "pull_request",
+ entries: docsEntries,
+ forceFullReasonCode: "override-variable",
+ })
+ assert.equal(variable.reasonCode, "override-variable")
+ })
+
+ it("ignores unknown override codes instead of failing open", () => {
+ const result = classifyEvent({ eventName: "pull_request", entries: docsEntries, forceFullReasonCode: "bogus" })
+ assert.equal(result.category, "docs-only")
+ })
+})
+
+describe("summary rendering", () => {
+ it("records the category, reason, and allowlist fingerprint", () => {
+ const summary = formatSummary(classifyEntries([{ status: "M", path: "docs/only.md" }]), {
+ eventName: "pull_request",
+ })
+ assert.match(summary, /docs-only/)
+ assert.match(summary, /Change-based validation classification/)
+ assert.ok(summary.includes(allowlistFingerprint()))
+ })
+})
+
+function git(tempRepo, args) {
+ return execFileSync("git", ["-c", "user.email=ci@example.com", "-c", "user.name=CI Test", ...args], {
+ cwd: tempRepo,
+ encoding: "utf8",
+ })
+}
+
+function createTempRepository() {
+ const tempRepo = fs.mkdtempSync(path.join(os.tmpdir(), "change-classification-"))
+ git(tempRepo, ["init", "--initial-branch=main"])
+ fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 1\n")
+ fs.mkdirSync(path.join(tempRepo, "docs"), { recursive: true })
+ fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "# base\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "base"])
+ return tempRepo
+}
+
+describe("history validation", () => {
+ it("classifies representative commits without errors", () => {
+ const tempRepo = createTempRepository()
+ try {
+ fs.writeFileSync(path.join(tempRepo, "docs/guide.md"), "# guide\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs: add guide (#101)"])
+
+ fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 2\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "feat: change code (#102)"])
+
+ fs.writeFileSync(path.join(tempRepo, "docs/mixed.md"), "# mixed\n")
+ fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 3\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "mixed: docs and code (#103)"])
+
+ const rows = runHistory(tempRepo, { limit: 10 })
+ const bySubject = new Map(rows.map((row) => [row.subject, row]))
+ assert.equal(bySubject.get("docs: add guide (#101)").category, "docs-only")
+ assert.equal(bySubject.get("feat: change code (#102)").category, "full")
+ assert.equal(bySubject.get("mixed: docs and code (#103)").category, "full")
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+})
+
+describe("ci command", () => {
+ function runCi(tempRepo, args) {
+ const githubOutput = path.join(tempRepo, "github-output.env")
+ const stepSummary = path.join(tempRepo, "step-summary.md")
+ const result = spawnSync(process.execPath, [classifierPath, "ci", ...args], {
+ cwd: repositoryRoot,
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ GITHUB_OUTPUT: githubOutput,
+ GITHUB_STEP_SUMMARY: stepSummary,
+ },
+ })
+ return { result, githubOutput, stepSummary }
+ }
+
+ function readOutputEnv(filePath) {
+ const entries = new Map()
+ for (const line of fs.readFileSync(filePath, "utf8").split("\n").filter(Boolean)) {
+ const index = line.indexOf("=")
+ entries.set(line.slice(0, index), line.slice(index + 1))
+ }
+ return entries
+ }
+
+ it("classifies a documentation-only merge commit and emits workflow outputs", () => {
+ const tempRepo = createTempRepository()
+ try {
+ git(tempRepo, ["checkout", "-b", "docs-change"])
+ fs.writeFileSync(path.join(tempRepo, "docs/pr.md"), "# pr\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs change"])
+ git(tempRepo, ["checkout", "main"])
+ git(tempRepo, ["merge", "--no-ff", "-m", "merge docs change", "docs-change"])
+ const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+
+ const { result, githubOutput, stepSummary } = runCi(tempRepo, [
+ "--event",
+ "pull_request",
+ "--head",
+ headSha,
+ "--repo-root",
+ tempRepo,
+ ])
+ assert.equal(result.status, 0, result.stderr)
+ const outputs = readOutputEnv(githubOutput)
+ assert.equal(outputs.get("category"), "docs-only")
+ assert.equal(outputs.get("changed-files"), "1")
+ const summary = fs.readFileSync(stepSummary, "utf8")
+ assert.match(summary, /docs-only/)
+ assert.ok(summary.includes("docs/pr.md"))
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+
+ for (const [flags, reason] of [
+ [["--force-full-label"], "override-label"],
+ [["--force-full-variable"], "override-variable"],
+ [["--force-full-label", "--force-full-variable"], "override-label"],
+ ]) {
+ it(`honors CLI overrides ${flags.join(" ")}`, () => {
+ const tempRepo = createTempRepository()
+ try {
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ const { result, githubOutput } = runCi(tempRepo, [
+ "--event",
+ "pull_request",
+ "--base",
+ base,
+ "--head",
+ head,
+ "--repo-root",
+ tempRepo,
+ ...flags,
+ ])
+ assert.equal(result.status, 0, result.stderr)
+ assert.equal(readOutputEnv(githubOutput).get("category"), "full")
+ assert.equal(readOutputEnv(githubOutput).get("reason-code"), reason)
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+ }
+ it("forces full validation for non-pull_request events", () => {
+ const tempRepo = createTempRepository()
+ try {
+ git(tempRepo, ["checkout", "-b", "docs-change"])
+ fs.writeFileSync(path.join(tempRepo, "docs/pr.md"), "# pr\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs change"])
+ git(tempRepo, ["checkout", "main"])
+ git(tempRepo, ["merge", "--no-ff", "-m", "merge docs change", "docs-change"])
+ const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+
+ const { result, githubOutput } = runCi(tempRepo, [
+ "--event",
+ "merge_group",
+ "--head",
+ headSha,
+ "--repo-root",
+ tempRepo,
+ ])
+ assert.equal(result.status, 0, result.stderr)
+ assert.equal(readOutputEnv(githubOutput).get("reason-code"), "non-pr-event")
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+
+ it("fails closed when the head commit is missing from history", () => {
+ const tempRepo = createTempRepository()
+ try {
+ const { result } = runCi(tempRepo, [
+ "--event",
+ "pull_request",
+ "--head",
+ "0".repeat(40),
+ "--repo-root",
+ tempRepo,
+ ])
+ assert.equal(result.status, 1)
+ assert.match(result.stderr, /::error/)
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+})
+
+describe("history report files", () => {
+ for (const filename of ["report.md", "report.txt", "report", "report.json"]) {
+ it(`preserves the Markdown report and JSON sidecar for ${filename}`, () => {
+ const tempRepo = createTempRepository()
+ try {
+ const output = path.join(tempRepo, filename)
+ const result = spawnSync(
+ process.execPath,
+ [classifierPath, "history", "--repo-root", tempRepo, "--output", output],
+ { encoding: "utf8" },
+ )
+ assert.equal(result.status, 0, result.stderr)
+ assert.match(fs.readFileSync(output, "utf8"), /Classified .*first-parent commit/)
+ const jsonPath = filename.endsWith(".md") ? output.replace(/\.md$/, ".json") : `${output}.json`
+ assert.deepEqual(JSON.parse(fs.readFileSync(jsonPath, "utf8")), [])
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+ }
+})
+
+describe("validation policy changes", () => {
+ for (const filePath of ["scripts/change-classification.mjs", "scripts/change-gate.mjs"]) {
+ for (const entry of [
+ { status: "M", path: filePath },
+ { status: "R", oldPath: filePath, path: "docs/policy.md" },
+ ]) {
+ it(`requires full validation for ${entry.status} ${filePath}`, () => {
+ const result = classifyEntries([entry])
+ assert.equal(result.category, "full")
+ assert.equal(result.reasonCode, "validation-policy-change")
+ })
+ }
+ }
+
+ it("independently detects policy changes even when the classifier reports docs-only", () => {
+ const tempRepo = createTempRepository()
+ try {
+ const baseSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ assert.equal(validationPolicyChanged(tempRepo, baseSha, baseSha), false)
+ fs.mkdirSync(path.join(tempRepo, "scripts"))
+ fs.writeFileSync(
+ path.join(tempRepo, "scripts/change-classification.mjs"),
+ 'console.log("Change classification: docs-only (allowlisted-documentation), 1 changed file(s)")',
+ )
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "widen classification"])
+ const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ assert.equal(
+ replayClassification(tempRepo, { baseSha, headSha, eventName: "pull_request" }).classification.category,
+ "docs-only",
+ )
+ assert.equal(validationPolicyChanged(tempRepo, baseSha, headSha), true)
+ assert.equal(validationPolicyChanged(tempRepo, "0".repeat(40), headSha), true)
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+
+ it("replays the complete explicit base range rather than just the last commit", () => {
+ const tempRepo = createTempRepository()
+ try {
+ fs.mkdirSync(path.join(tempRepo, "scripts"))
+ fs.copyFileSync(classifierPath, path.join(tempRepo, "scripts/change-classification.mjs"))
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "install classifier"])
+ const baseSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 2\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "code change"])
+ fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "# updated\n")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs change"])
+ const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ const replay = replayClassification(tempRepo, { baseSha, headSha, eventName: "pull_request" })
+ assert.equal(replay.status, 0)
+ assert.equal(replay.classification.category, "full")
+ assert.equal(replay.classification.changedFiles, 2)
+ assert.equal(validationPolicyChanged(tempRepo, baseSha, headSha), false)
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+})
+
+it("escapes filenames in classification summaries", () => {
+ const filename = "docs/`|\nforged.md"
+ const summary = formatSummary(classifyEntries([{ status: "M", path: filename }]))
+ assert.ok(summary.includes("docs/`<details>|
forged.md"))
+ assert.ok(!summary.includes(filename))
+})
+
+it("pins the complete allowlist fingerprint including protected segments", () => {
+ assert.equal(allowlistFingerprint(), "15dc52b392ab")
+})
+
+for (const sha of ["", "abc", "g".repeat(40), "a".repeat(39), "a".repeat(41)]) {
+ it(`rejects invalid SHA ${sha}`, () => {
+ assert.throws(() => resolveCommits(repositoryRoot, sha, "a".repeat(40)), /40-character/)
+ assert.throws(() => resolveCommits(repositoryRoot, "a".repeat(40), sha), /40-character/)
+ assert.equal(validationPolicyChanged(repositoryRoot, sha, "a".repeat(40)), true)
+ assert.equal(validationPolicyChanged(repositoryRoot, "a".repeat(40), sha), true)
+ })
+}
+
+it("runs the gate CLI against a docs-only repository and writes its summary", () => {
+ const tempRepo = createTempRepository()
+ try {
+ fs.mkdirSync(path.join(tempRepo, "scripts"))
+ for (const file of ["change-classification.mjs", "change-gate.mjs"])
+ fs.copyFileSync(path.join(repositoryRoot, "scripts", file), path.join(tempRepo, "scripts", file))
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "install validation scripts"])
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ const cases = [
+ { name: "docs skip", actual: "skipped", status: 0 },
+ { name: "unexpected run", actual: "success", status: 1 },
+ { name: "label", actual: "success", category: "full", reason: "override-label", label: "true", status: 0 },
+ {
+ name: "variable",
+ actual: "success",
+ category: "full",
+ reason: "override-variable",
+ variable: "true",
+ status: 0,
+ },
+ {
+ name: "label wins",
+ actual: "success",
+ category: "full",
+ reason: "override-label",
+ label: "true",
+ variable: "true",
+ status: 0,
+ },
+ {
+ name: "draft",
+ actual: "skipped",
+ rule: "pr-non-draft",
+ draft: "true",
+ category: "full",
+ reason: "override-label",
+ label: "true",
+ status: 0,
+ },
+ {
+ name: "ready",
+ actual: "success",
+ rule: "pr-non-draft",
+ category: "full",
+ reason: "override-label",
+ label: "true",
+ status: 0,
+ },
+ { name: "missing event", actual: "skipped", event: "", error: /EVENT_NAME is required/, status: 1 },
+ { name: "missing watches", actual: "skipped", watch: "", error: /WATCHED_JOBS is empty/, status: 1 },
+ { name: "missing needs", actual: "skipped", needs: "{}", error: /missing/, status: 1 },
+ { name: "missing base", actual: "skipped", base: "", error: /--base and --head are required/, status: 1 },
+ ]
+ for (const testCase of cases) {
+ const { actual, status } = testCase
+ const summary = path.join(tempRepo, `summary-${testCase.name}.md`)
+ const result = spawnSync(
+ process.execPath,
+ [
+ path.join(tempRepo, "scripts/change-gate.mjs"),
+ "ci",
+ "--base",
+ testCase.base ?? base,
+ "--head",
+ head,
+ "--skip-metrics",
+ ],
+ {
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ EVENT_NAME: testCase.event ?? "pull_request",
+ WATCHED_JOBS: testCase.watch ?? `expensive:${testCase.rule ?? "always"}:gated`,
+ NEEDS_JSON:
+ testCase.needs ??
+ JSON.stringify({
+ classify: {
+ result: "success",
+ outputs: {
+ category: testCase.category ?? "docs-only",
+ "reason-code": testCase.reason ?? "allowlisted-documentation",
+ },
+ },
+ expensive: { result: actual },
+ }),
+ GITHUB_STEP_SUMMARY: summary,
+ FORCE_FULL_LABEL: testCase.label ?? "false",
+ PR_DRAFT: testCase.draft ?? "false",
+ FORCE_FULL_VARIABLE: testCase.variable ?? "false",
+ },
+ },
+ )
+ assert.equal(result.status, status, `${testCase.name}: ${result.stderr}`)
+ if (testCase.error) {
+ assert.match(result.stderr, testCase.error)
+ continue
+ }
+ assert.match(fs.readFileSync(summary, "utf8"), /Validation gate/)
+ assert.match(fs.readFileSync(summary, "utf8"), /Runner-minute metrics were unavailable/)
+ }
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+})
+
+for (const source of ["process.exit(1)", 'console.log("malformed")']) {
+ it(`rejects a failed or malformed classifier replay: ${source}`, () => {
+ const tempRepo = createTempRepository()
+ try {
+ fs.mkdirSync(path.join(tempRepo, "scripts"))
+ fs.writeFileSync(path.join(tempRepo, "scripts/change-classification.mjs"), source)
+ assert.deepEqual(
+ replayClassification(tempRepo, {
+ baseSha: "a".repeat(40),
+ headSha: "b".repeat(40),
+ eventName: "pull_request",
+ }),
+ { status: 1, classification: undefined },
+ )
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+}
+
+it("uses the PR merge base and detects documentation renames", () => {
+ const tempRepo = createTempRepository()
+ try {
+ git(tempRepo, ["checkout", "-b", "pr"])
+ git(tempRepo, ["mv", "docs/base.md", "docs/renamed.md"])
+ git(tempRepo, ["commit", "-m", "rename docs"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ git(tempRepo, ["checkout", "main"])
+ fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "main-only change")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "advance main"])
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ assert.deepEqual(changedEntries(tempRepo, base, head).entries, [
+ { status: "R", oldPath: "docs/base.md", path: "docs/renamed.md" },
+ ])
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+})
+
+for (const filename of ["docs/space name.md", "docs/日本語.md", "docs/line\nbreak.md", "docs/UPPER.MD"]) {
+ it(`classifies the real git path ${JSON.stringify(filename)}`, () => {
+ const tempRepo = createTempRepository()
+ try {
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.writeFileSync(path.join(tempRepo, filename), "# documentation")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "add unusual path"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ const { entries } = changedEntries(tempRepo, base, head)
+ assert.deepEqual(entries, [{ status: "A", path: filename }])
+ assert.equal(classifyEntries(entries).category, filename.endsWith(".MD") ? "full" : "docs-only")
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+ })
+}
+
+it("requires full validation for a real documentation-to-source rename", () => {
+ const tempRepo = createTempRepository()
+ try {
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.mkdirSync(path.join(tempRepo, "src"))
+ git(tempRepo, ["mv", "docs/base.md", "src/source.md"])
+ git(tempRepo, ["commit", "-m", "move documentation into source"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ const { entries } = changedEntries(tempRepo, base, head)
+ assert.deepEqual(entries, [{ status: "R", oldPath: "docs/base.md", path: "src/source.md" }])
+ assert.equal(classifyEntries(entries).category, "full")
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ }
+})
+
+it("fails closed when a shallow clone lacks the base commit", () => {
+ const tempRepo = createTempRepository()
+ const clone = fs.mkdtempSync(path.join(os.tmpdir(), "classification-shallow-"))
+ try {
+ const base = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated")
+ git(tempRepo, ["add", "."])
+ git(tempRepo, ["commit", "-m", "docs"])
+ const head = git(tempRepo, ["rev-parse", "HEAD"]).trim()
+ git(tempRepo, ["clone", "--depth=1", `file://${tempRepo}`, clone])
+ assert.equal(git(clone, ["rev-parse", "--is-shallow-repository"]).trim(), "true")
+ assert.throws(() => changedEntries(clone, base, head), /Could not resolve base commit/)
+ assert.equal(validationPolicyChanged(clone, base, head), true)
+ } finally {
+ fs.rmSync(tempRepo, { recursive: true, force: true })
+ fs.rmSync(clone, { recursive: true, force: true })
+ }
+})
+
+it("renders formatting and autolink punctuation as literal filename characters", () => {
+ const filename = "docs/[link](https://example.com)*_~#@!\\.md"
+ const summary = formatSummary(classifyEntries([{ status: "M", path: filename }]))
+ const line = summary.split("\n").find((line) => line.startsWith("- docs/"))
+ assert.equal(
+ line,
+ "- docs/[link](https://example.com)*_~#@!\.md",
+ )
+})
+
+it("caps long filename summaries", () => {
+ const filename = `docs/${"a".repeat(600)}.md`
+ const summary = formatSummary(classifyEntries([{ status: "M", path: filename }]))
+ const line = summary.split("\n").find((line) => line.startsWith("- docs/"))
+ assert.equal(line, `- docs/${"a".repeat(495)}…`)
+})
diff --git a/scripts/change-gate.mjs b/scripts/change-gate.mjs
new file mode 100644
index 0000000000..124013b11f
--- /dev/null
+++ b/scripts/change-gate.mjs
@@ -0,0 +1,376 @@
+#!/usr/bin/env node
+
+// Always-running validation gate for change-based CI (issue #2291).
+//
+// The gate runs after the classification and every validation job of its
+// workflow and rejects incoherent outcomes:
+// - a missing, malformed, or non-terminal classification,
+// - validation jobs that did not run when full validation was required,
+// - validation jobs that ran when a documentation-only skip was expected,
+// - a classifier replay that disagrees with the recorded classification.
+//
+// It also records the observability contract: classified category, the
+// skipped work with explanations, runner minutes per job, and the time
+// until the gate completed.
+
+import { spawnSync } from "node:child_process"
+import fs from "node:fs"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+
+const WATCH_RULES = new Set(["always", "pr-only", "pr-non-draft"])
+const TERMINAL_RESULTS = new Set(["success", "failure", "skipped", "cancelled"])
+
+export const SKIP_EXPLANATION =
+ "Documentation-only change: every changed file matches the allowlist, so this job skipped expensive validation."
+export const FALLBACK_EXPLANATION = "Classification was unavailable, so the fail-safe default selected full validation."
+
+// `name:rule:gated` triples describing the jobs this gate watches.
+export function parseWatchSpec(spec) {
+ if (typeof spec !== "string" || spec.trim().length === 0) throw new Error("WATCHED_JOBS is empty")
+ return spec.split(",").map((entry) => {
+ const parts = entry.trim().split(":")
+ if (parts.length !== 3) throw new Error(`Malformed watch entry: ${entry}`)
+ const [name, rule, gated] = parts
+ if (!WATCH_RULES.has(rule)) throw new Error(`Unknown watch rule in entry: ${entry}`)
+ if (gated !== "gated" && gated !== "ungated") throw new Error(`Unknown gating flag in entry: ${entry}`)
+ return { name, rule, gated: gated === "gated" }
+ })
+}
+
+export function isDocsOnlySkip(needsEntry, eventName) {
+ return (
+ needsEntry?.result === "success" &&
+ needsEntry?.outputs?.category === "docs-only" &&
+ eventName === "pull_request"
+ )
+}
+
+export function expectedResult({ rule, gated }, { eventName, prDraft, docsOnly }) {
+ if (rule === "pr-only" && eventName !== "pull_request") return "skipped"
+ if (rule === "pr-non-draft" && eventName === "pull_request" && prDraft) return "skipped"
+ if (docsOnly && gated && eventName === "pull_request") return "skipped"
+ return "success"
+}
+
+function explanationFor(expected, { docsOnly, classificationFailed }) {
+ if (expected === "skipped") return docsOnly ? SKIP_EXPLANATION : "Job has its own event or draft condition."
+ if (classificationFailed) return FALLBACK_EXPLANATION
+ return "Full validation ran."
+}
+
+// Compares the needs graph with the outcome implied by the
+// classification. Missing results, failed full validation, and jobs
+// whose skip state contradicts the classification are all violations.
+export function evaluateGate({ needs, watched, eventName, prDraft, classifierReplay, policyChanged = false }) {
+ const violations = []
+ const notes = []
+
+ const classify = needs?.classify
+ if (!classify || typeof classify.result !== "string") {
+ violations.push("Classification job result is missing from the needs graph.")
+ } else if (!TERMINAL_RESULTS.has(classify.result)) {
+ violations.push(`Classification job reported non-terminal result: ${classify.result}`)
+ } else if (classify.result !== "success" && classify.result !== "failure") {
+ violations.push(`Classification must either succeed or fail; got: ${classify.result}`)
+ }
+
+ const classificationFailed = classify?.result === "failure"
+ if (classify?.result === "success") {
+ const category = classify.outputs?.category
+ if (category !== "docs-only" && category !== "full") {
+ violations.push(`Classification output category is malformed: ${category}`)
+ }
+ }
+
+ const docsOnly = isDocsOnlySkip(classify, eventName)
+ if (docsOnly && policyChanged) {
+ violations.push(
+ "Validation policy changed or could not be verified against the base; full validation is required.",
+ )
+ }
+
+ if (classifierReplay) {
+ if (classifierReplay.status === 0) {
+ const replay = classifierReplay.classification
+ if (classify?.result === "failure") {
+ violations.push(
+ "The gate reproduced a classification the classify job reported as failed; the runs are inconsistent.",
+ )
+ } else if (
+ replay?.category !== classify?.outputs?.category ||
+ replay?.reasonCode !== classify?.outputs?.["reason-code"]
+ ) {
+ violations.push(
+ `Classifier replay disagrees with the recorded classification: replay=${replay?.category}/${replay?.reasonCode} recorded=${classify?.outputs?.category}/${classify?.outputs?.["reason-code"]}`,
+ )
+ }
+ } else if (classify?.result === "success") {
+ violations.push(
+ "The gate could not reproduce the classification that the classify job reported; refusing to trust the recorded category.",
+ )
+ } else {
+ notes.push(
+ "Both the classify job and the gate replay failed; full validation ran by the fail-safe default.",
+ )
+ }
+ }
+
+ const rows = []
+ for (const job of watched) {
+ const actual = needs?.[job.name]?.result
+ const expected = expectedResult(job, { eventName, prDraft, docsOnly })
+ const ok = actual === expected
+ if (!actual) {
+ violations.push(`Watched job ${job.name} has no result in the needs graph (missing or misnamed).`)
+ } else if (!ok) {
+ violations.push(
+ expected === "success"
+ ? `Required validation missing: ${job.name} reported "${actual}" but full validation was required.`
+ : `Unexpected run: ${job.name} reported "${actual}" but the classification expected it to be skipped.`,
+ )
+ }
+ rows.push({
+ ...job,
+ expected,
+ actual: actual ?? "missing",
+ ok,
+ explanation: explanationFor(expected, { docsOnly, classificationFailed }),
+ })
+ }
+
+ return { ok: violations.length === 0, violations, notes, rows, docsOnly, classificationFailed }
+}
+
+// Best-effort runner-minute metrics from the Actions API. `fetchJson`
+// is injectable for tests.
+export function collectJobMetrics({ repository, runId, fetchJson }) {
+ const run = fetchJson(`repos/${repository}/actions/runs/${runId}`)
+ const jobs = []
+ let page = 1
+ for (;;) {
+ const response = fetchJson(`repos/${repository}/actions/runs/${runId}/jobs?per_page=100&page=${page}`)
+ jobs.push(...(response.jobs ?? []))
+ if ((response.jobs ?? []).length < 100) break
+ page += 1
+ }
+
+ const minutes = (start, end) => Math.max(0, (new Date(end) - new Date(start)) / 60_000)
+ const perJob = jobs
+ .filter((job) => job.started_at && job.completed_at)
+ .map((job) => ({ name: job.name, runnerMinutes: Number(minutes(job.started_at, job.completed_at).toFixed(2)) }))
+ const totalRunnerMinutes = Number(perJob.reduce((sum, job) => sum + job.runnerMinutes, 0).toFixed(2))
+ const latestCompletion = jobs
+ .map((job) => job.completed_at)
+ .filter(Boolean)
+ .sort()
+ .at(-1)
+ const completionMinutes =
+ run?.started_at && latestCompletion ? Number(minutes(run.started_at, latestCompletion).toFixed(2)) : undefined
+
+ return { perJob, totalRunnerMinutes, completionMinutes }
+}
+
+function escapeSummaryCell(value) {
+ return String(value).replaceAll("|", "\\|").replaceAll("\n", " ")
+}
+
+export function renderGateSummary({ rows, needs, eventName, metrics, violations, notes }) {
+ const classify = needs?.classify
+ const lines = [
+ "## Validation gate",
+ "",
+ "| Field | Value |",
+ "| --- | --- |",
+ `| Event | ${escapeSummaryCell(eventName)} |`,
+ `| Classification | ${escapeSummaryCell(classify?.outputs?.category ?? "unavailable")} (${escapeSummaryCell(classify?.outputs?.["reason-code"] ?? "no reason recorded")}) |`,
+ `| Classification job | ${escapeSummaryCell(classify?.result ?? "missing")} |`,
+ `| Skip decision | ${rows.some((row) => row.expected === "skipped" && row.explanation === SKIP_EXPLANATION) ? "documentation-only jobs skipped" : "no classification-based skips"} |`,
+ `| Gate outcome | ${violations.length === 0 ? "passed" : "failed"} |`,
+ "",
+ "| Job | Watch rule | Gated | Expected | Actual | Explanation |",
+ "| --- | --- | --- | --- | --- | --- |",
+ ]
+ for (const row of rows) {
+ lines.push(
+ `| ${escapeSummaryCell(row.name)} | ${row.rule} | ${row.gated ? "yes" : "no"} | ${row.expected} | ${escapeSummaryCell(row.actual)} | ${escapeSummaryCell(row.explanation)} |`,
+ )
+ }
+
+ if (metrics) {
+ lines.push(
+ "",
+ "| Runner metric | Minutes |",
+ "| --- | ---: |",
+ `| Total runner minutes (this workflow run) | ${metrics.totalRunnerMinutes} |`,
+ ...(metrics.completionMinutes === undefined
+ ? []
+ : [`| Required-check completion time (run start to last job) | ${metrics.completionMinutes} |`]),
+ "",
+ "Runner minutes per job
",
+ "",
+ "| Job | Minutes |",
+ "| --- | ---: |",
+ ...metrics.perJob.map((job) => `| ${escapeSummaryCell(job.name)} | ${job.runnerMinutes} |`),
+ "",
+ " ",
+ )
+ } else {
+ lines.push("", "Runner-minute metrics were unavailable for this run (best effort only).")
+ }
+
+ if (notes.length > 0) lines.push("", ...notes.map((note) => `Note: ${note}`))
+ if (violations.length > 0) {
+ lines.push("", "### Gate violations", "")
+ for (const violation of violations) lines.push(`- ${violation}`)
+ }
+ return `${lines.join("\n")}\n`
+}
+
+function repositoryRoot() {
+ return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
+}
+
+export function parseClassifierStdout(stdout) {
+ const match = /Change classification: (\S+) \(([^)]+)\), (\d+) changed file/.exec(stdout ?? "")
+ return match ? { category: match[1], reasonCode: match[2], changedFiles: Number(match[3]) } : undefined
+}
+
+// Check git directly, independently of the head classifier's allowlist or replay.
+// Missing history and git errors fail safe by refusing documentation-only skips.
+export function validationPolicyChanged(repoRoot, baseSha, headSha) {
+ if (![baseSha, headSha].every((sha) => /^[0-9a-f]{40}$/i.test(sha ?? ""))) return true
+ const result = spawnSync(
+ "git",
+ [
+ "diff",
+ "--quiet",
+ `${baseSha}...${headSha}`,
+ "--",
+ "scripts/change-classification.mjs",
+ "scripts/change-gate.mjs",
+ ],
+ { cwd: repoRoot },
+ )
+ return Boolean(result.error) || result.status !== 0
+}
+
+export function replayClassification(repoRoot, { baseSha, headSha, eventName, forceFullLabel, forceFullVariable }) {
+ const args = [
+ path.join(repoRoot, "scripts/change-classification.mjs"),
+ "ci",
+ "--event",
+ eventName,
+ "--base",
+ baseSha,
+ "--head",
+ headSha,
+ "--repo-root",
+ repoRoot,
+ ]
+ if (forceFullLabel === "true") args.push("--force-full-label")
+ if (forceFullVariable === "true") args.push("--force-full-variable")
+
+ const result = spawnSync(process.execPath, args, {
+ cwd: repoRoot,
+ encoding: "utf8",
+ // The replay must not write workflow outputs or summaries.
+ env: { ...process.env, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" },
+ })
+ if (result.status !== 0) return { status: result.status ?? 1, classification: undefined }
+
+ const classification = parseClassifierStdout(result.stdout)
+ return classification ? { status: 0, classification } : { status: 1, classification: undefined }
+}
+
+function argument(name) {
+ const index = process.argv.indexOf(name)
+ return index === -1 ? undefined : process.argv[index + 1]
+}
+
+function main() {
+ const repoRoot = repositoryRoot()
+ const command = process.argv[2]
+ if (command !== "ci") throw new Error("Usage: change-gate.mjs ci --base --head [--skip-metrics]")
+
+ const headSha = argument("--head")
+ const baseSha = argument("--base")
+ if (!headSha || !baseSha) throw new Error("--base and --head are required")
+
+ const eventName = process.env.EVENT_NAME
+ if (!eventName) throw new Error("EVENT_NAME is required")
+ const watched = parseWatchSpec(process.env.WATCHED_JOBS ?? "")
+ const needs = JSON.parse(process.env.NEEDS_JSON ?? "{}")
+
+ const classifierReplay = replayClassification(repoRoot, {
+ baseSha,
+ headSha,
+ eventName,
+ forceFullLabel: process.env.FORCE_FULL_LABEL,
+ forceFullVariable: process.env.FORCE_FULL_VARIABLE,
+ })
+ if (classifierReplay.classification === undefined && classifierReplay.status === 0) {
+ classifierReplay.status = 1
+ }
+
+ const evaluation = evaluateGate({
+ needs,
+ watched,
+ eventName,
+ prDraft: process.env.PR_DRAFT === "true",
+ policyChanged: validationPolicyChanged(repoRoot, baseSha, headSha),
+ classifierReplay: { status: classifierReplay.status, classification: classifierReplay.classification },
+ })
+
+ let metrics
+ if (!process.argv.includes("--skip-metrics") && process.env.GH_TOKEN && process.env.RUN_ID) {
+ try {
+ metrics = collectJobMetrics({
+ repository: process.env.GITHUB_REPOSITORY,
+ runId: process.env.RUN_ID,
+ fetchJson: (route) => {
+ const result = spawnSync("gh", ["api", route], {
+ encoding: "utf8",
+ env: process.env,
+ maxBuffer: 20 * 1024 * 1024,
+ })
+ if (result.status !== 0) throw new Error(result.stderr)
+ return JSON.parse(result.stdout)
+ },
+ })
+ } catch (error) {
+ console.warn(
+ `::warning title=Validation gate::Runner-minute metrics unavailable: ${String(error.message).replaceAll("\n", " ")}`,
+ )
+ }
+ }
+
+ if (process.env.GITHUB_STEP_SUMMARY) {
+ fs.appendFileSync(
+ process.env.GITHUB_STEP_SUMMARY,
+ renderGateSummary({ ...evaluation, needs, eventName, metrics }),
+ )
+ }
+
+ for (const note of evaluation.notes) console.log(`Validation gate note: ${note}`)
+ for (const violation of evaluation.violations) {
+ console.error(`::error title=Validation gate::${violation.replaceAll("\n", " ")}`)
+ }
+ if (evaluation.ok) {
+ console.log(
+ `Validation gate passed: classification=${needs?.classify?.outputs?.category ?? "unavailable"}, watched=${evaluation.rows.length} job(s).`,
+ )
+ return
+ }
+ process.exitCode = 1
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ try {
+ main()
+ } catch (error) {
+ console.error(`::error title=Validation gate::${String(error.message).replaceAll("\n", " ")}`)
+ console.error(`Validation gate failed: ${error.message}`)
+ process.exitCode = 1
+ }
+}
diff --git a/scripts/change-gate.test.mjs b/scripts/change-gate.test.mjs
new file mode 100644
index 0000000000..fa5448975a
--- /dev/null
+++ b/scripts/change-gate.test.mjs
@@ -0,0 +1,544 @@
+import assert from "node:assert/strict"
+import { parse } from "yaml"
+import fs from "node:fs"
+import path from "node:path"
+import { describe, it } from "node:test"
+import { fileURLToPath } from "node:url"
+
+const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
+
+import {
+ collectJobMetrics,
+ evaluateGate,
+ expectedResult,
+ parseWatchSpec,
+ parseClassifierStdout,
+ renderGateSummary,
+} from "./change-gate.mjs"
+
+const needsFor = (results, classify) => ({
+ classify: classify ?? { result: "success", outputs: { category: "full", "reason-code": "non-documentation-path" } },
+ ...Object.fromEntries(Object.entries(results).map(([name, result]) => [name, { result }])),
+})
+
+describe("watch specification", () => {
+ it("parses name:rule:gated triples", () => {
+ assert.deepEqual(parseWatchSpec("job-a:always:gated, job-b:pr-only:ungated, job-c:pr-non-draft:gated"), [
+ { name: "job-a", rule: "always", gated: true },
+ { name: "job-b", rule: "pr-only", gated: false },
+ { name: "job-c", rule: "pr-non-draft", gated: true },
+ ])
+ })
+
+ it("rejects malformed specifications", () => {
+ assert.throws(() => parseWatchSpec("job-a:always"))
+ assert.throws(() => parseWatchSpec("job-a:sometimes:gated"))
+ assert.throws(() => parseWatchSpec("job-a:always:maybe"))
+ assert.throws(() => parseWatchSpec(""))
+ })
+})
+
+describe("expected job outcomes", () => {
+ const base = { eventName: "pull_request", prDraft: false, docsOnly: false }
+
+ it("requires success for always-running jobs on pull requests", () => {
+ assert.equal(expectedResult({ rule: "always", gated: true }, base), "success")
+ })
+
+ it("skips gated jobs only for documentation-only pull request classifications", () => {
+ assert.equal(expectedResult({ rule: "always", gated: true }, { ...base, docsOnly: true }), "skipped")
+ })
+
+ it("never skips on the merge queue even when a docs-only classification is recorded", () => {
+ assert.equal(
+ expectedResult({ rule: "always", gated: true }, { ...base, eventName: "merge_group", docsOnly: true }),
+ "success",
+ )
+ })
+
+ it("expect pull-request-only jobs to be skipped on other events", () => {
+ assert.equal(expectedResult({ rule: "pr-only", gated: false }, { ...base, eventName: "push" }), "skipped")
+ assert.equal(expectedResult({ rule: "pr-only", gated: true }, { ...base, eventName: "merge_group" }), "skipped")
+ })
+
+ it("expect non-draft mutations to be skipped for drafts but not for the merge queue", () => {
+ assert.equal(expectedResult({ rule: "pr-non-draft", gated: true }, { ...base, prDraft: true }), "skipped")
+ assert.equal(
+ expectedResult({ rule: "pr-non-draft", gated: true }, { ...base, eventName: "merge_group" }),
+ "success",
+ )
+ })
+})
+
+describe("gate evaluation", () => {
+ const watched = [
+ { name: "cheap", rule: "always", gated: false },
+ { name: "expensive", rule: "always", gated: true },
+ ]
+
+ for (const result of ["skipped", "cancelled", "queued", undefined]) {
+ it(`rejects classification result ${result}`, () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }, { result }),
+ watched,
+ eventName: "pull_request",
+ })
+ assert.equal(evaluation.ok, false)
+ assert.match(
+ evaluation.violations.join("; "),
+ result === undefined
+ ? /result is missing/
+ : result === "queued"
+ ? /non-terminal result/
+ : /must either succeed or fail/,
+ )
+ })
+ }
+ it("does not trust docs-only outputs leaked by a failed classifier", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ { result: "failure", outputs: { category: "docs-only" } },
+ ),
+ watched,
+ eventName: "pull_request",
+ })
+ assert.equal(evaluation.ok, false)
+ assert.equal(evaluation.docsOnly, false)
+ })
+ it("passes when full validation ran for a code change", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(evaluation.ok, evaluation.violations.join("; "))
+ })
+
+ it("passes when gated jobs skip a documentation-only change", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ {
+ result: "success",
+ outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(evaluation.ok, evaluation.violations.join("; "))
+ assert.ok(evaluation.rows.find((row) => row.name === "expensive").explanation.includes("Documentation-only"))
+ })
+
+ it("rejects a docs-only result when validation policy changed despite a matching replay", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ { result: "success", outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" } },
+ ),
+ watched,
+ eventName: "pull_request",
+ policyChanged: true,
+ classifierReplay: {
+ status: 0,
+ classification: { category: "docs-only", reasonCode: "allowlisted-documentation" },
+ },
+ })
+ assert.equal(evaluation.ok, false)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("Validation policy changed")))
+ })
+
+ it("accepts policy changes when full validation ran", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }),
+ watched,
+ eventName: "pull_request",
+ policyChanged: true,
+ })
+ assert.equal(evaluation.ok, true)
+ })
+
+ it("rejects a gated job that ran despite a docs-only classification", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "success" },
+ {
+ result: "success",
+ outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("expected it to be skipped")))
+ })
+
+ it("rejects missing or failed required validation", () => {
+ for (const actual of ["skipped", "failure", "cancelled"]) {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: actual }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok, `actual=${actual} must violate the gate`)
+ }
+ })
+
+ it("rejects a missing classification job", () => {
+ const evaluation = evaluateGate({
+ needs: { cheap: { result: "success" }, expensive: { result: "success" } },
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("missing")))
+ })
+
+ it("rejects a malformed classification category", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "success" },
+ {
+ result: "success",
+ outputs: { category: "banana", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("malformed")))
+ })
+
+ it("accepts the fail-safe fallback when classification failed and full validation ran", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }, { result: "failure" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(evaluation.ok, evaluation.violations.join("; "))
+ assert.ok(evaluation.rows.every((row) => row.explanation.includes("fail-safe")))
+ })
+
+ it("rejects a skipped job after a failed classification", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "skipped" }, { result: "failure" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("Required validation missing")))
+ })
+
+ it("validates the classification with a matching deterministic replay", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ {
+ result: "success",
+ outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ classifierReplay: {
+ status: 0,
+ classification: { category: "docs-only", reasonCode: "allowlisted-documentation", changedFiles: "1" },
+ },
+ })
+ assert.ok(evaluation.ok, evaluation.violations.join("; "))
+ })
+
+ for (const classification of [
+ { category: "full", reasonCode: "different" },
+ { category: "docs-only", reasonCode: "non-documentation-path" },
+ ]) {
+ it(`rejects independent replay mismatch ${JSON.stringify(classification)}`, () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ classifierReplay: {
+ status: 0,
+ classification: classification,
+ },
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("replay disagrees")))
+ })
+ }
+
+ it("rejects a replay failure when the classify job claimed success", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success", expensive: "success" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ classifierReplay: { status: 1 },
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("could not reproduce")))
+ })
+
+ it("reports watched jobs that are missing from the needs graph", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor({ cheap: "success" }),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ assert.ok(!evaluation.ok)
+ assert.ok(evaluation.violations.some((violation) => violation.includes("expensive")))
+ })
+})
+
+describe("runner-minute metrics", () => {
+ it("sums job minutes and reports completion time from injected API pages", () => {
+ const metrics = collectJobMetrics({
+ repository: "Zoo-Code-Org/Zoo-Code",
+ runId: 42,
+ fetchJson: (route) => {
+ if (route.endsWith("/jobs?per_page=100&page=1")) {
+ return {
+ jobs: [
+ { name: "a", started_at: "2026-01-01T00:00:00Z", completed_at: "2026-01-01T00:10:00Z" },
+ { name: "b", started_at: "2026-01-01T00:00:00Z", completed_at: "2026-01-01T00:05:30Z" },
+ ],
+ }
+ }
+ if (route === "repos/Zoo-Code-Org/Zoo-Code/actions/runs/42") {
+ return { started_at: "2026-01-01T00:00:00Z" }
+ }
+ throw new Error(`unexpected route: ${route}`)
+ },
+ })
+ assert.equal(metrics.totalRunnerMinutes, 15.5)
+ assert.equal(metrics.completionMinutes, 10)
+ })
+})
+
+describe("summary rendering", () => {
+ const watched = [
+ { name: "cheap", rule: "always", gated: false },
+ { name: "expensive", rule: "always", gated: true },
+ ]
+
+ it("renders the job outcome table, metrics, and violations", () => {
+ const evaluation = evaluateGate({
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ {
+ result: "success",
+ outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ watched,
+ eventName: "pull_request",
+ prDraft: false,
+ })
+ const summary = renderGateSummary({
+ ...evaluation,
+ needs: needsFor(
+ { cheap: "success", expensive: "skipped" },
+ {
+ result: "success",
+ outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" },
+ },
+ ),
+ eventName: "pull_request",
+ metrics: { perJob: [{ name: "cheap", runnerMinutes: 1.5 }], totalRunnerMinutes: 1.5, completionMinutes: 4 },
+ violations: evaluation.violations,
+ })
+ assert.match(summary, /Validation gate/)
+ assert.match(summary, /docs-only/)
+ assert.match(summary, /Total runner minutes/)
+ assert.match(summary, /expensive/)
+ })
+})
+
+// --- Workflow wiring assertions -------------------------------------------
+
+const GATING_TERMS = [
+ "!cancelled()",
+ "needs.classify.result != 'success'",
+ "needs.classify.outputs.category != 'docs-only'",
+]
+
+const WORKFLOW_EXPECTATIONS = [
+ {
+ file: "code-qa.yml",
+ hasMergeGroup: true,
+ jobs: [
+ { id: "dependency-review", rule: "pr-only", gated: false },
+ { id: "invisible-chars", rule: "always", gated: false },
+ { id: "format-check", rule: "always", gated: false },
+ { id: "check-translations", rule: "always", gated: true },
+ { id: "knip", rule: "always", gated: false },
+ { id: "compile", rule: "always", gated: false },
+ { id: "build-vsix", rule: "pr-only", gated: true, prOnlyCondition: true },
+ { id: "unit-test", rule: "always", gated: false },
+ ],
+ },
+ {
+ file: "e2e.yml",
+ hasMergeGroup: true,
+ jobs: [{ id: "e2e-mock", rule: "always", gated: true }],
+ },
+ {
+ file: "visual-regression.yml",
+ hasMergeGroup: true,
+ jobs: [
+ { id: "webview-visual", rule: "always", gated: true },
+ { id: "theme-fixtures", rule: "always", gated: true },
+ { id: "extension-host-visual", rule: "always", gated: true },
+ ],
+ },
+ {
+ file: "mutation-testing.yml",
+ hasMergeGroup: true,
+ jobs: [{ id: "mutation-diff", rule: "pr-non-draft", gated: true, prOnlyCondition: false }],
+ },
+ {
+ file: "codeql.yml",
+ hasMergeGroup: false,
+ jobs: [{ id: "analyze", rule: "always", gated: true }],
+ },
+]
+
+describe("change-based validation workflow wiring", () => {
+ for (const expectation of WORKFLOW_EXPECTATIONS) {
+ it(`validates ${expectation.file}`, () => {
+ const workflow = parse(
+ fs.readFileSync(path.join(repositoryRoot, ".github/workflows", expectation.file), "utf8"),
+ )
+ assert.ok(workflow.on.pull_request.types.includes("labeled"))
+ assert.ok(workflow.on.pull_request.types.includes("unlabeled"))
+ assert.equal(workflow.on.pull_request.paths, undefined)
+ assert.equal(workflow.on.pull_request["paths-ignore"], undefined)
+ assert.equal(Boolean(workflow.on.merge_group), expectation.hasMergeGroup)
+ const jobs = workflow.jobs
+ assert.deepEqual(
+ Object.keys(jobs).sort(),
+ ["classify", "validation-gate", ...expectation.jobs.map((job) => job.id)].sort(),
+ )
+ const gate = jobs["validation-gate"]
+ assert.equal(gate.if, "always()")
+ assert.deepEqual([...gate.needs].sort(), ["classify", ...expectation.jobs.map((job) => job.id)].sort())
+ const step = gate.steps.find((step) => step.name === "Run validation gate")
+ assert.equal(
+ step.env.WATCHED_JOBS,
+ expectation.jobs.map((job) => `${job.id}:${job.rule}:${job.gated ? "gated" : "ungated"}`).join(","),
+ )
+ assert.equal(step.env.NEEDS_JSON, "${{ toJSON(needs) }}")
+ assert.equal(step.env.EVENT_NAME, "${{ github.event_name }}")
+ assert.match(step.env.FORCE_FULL_LABEL, /ci:full-validation/)
+ assert.match(step.env.FORCE_FULL_VARIABLE, /ZOO_CODE_REQUIRE_FULL_VALIDATION/)
+ const classify = jobs.classify
+ const classifyStep = classify.steps.find((step) => step.id === "classify")
+ assert.equal(classify.steps[0].with["fetch-depth"], 0)
+ assert.equal(classifyStep.uses, "./.github/actions/classify-changes")
+ assert.equal(classifyStep.with["event-name"], "${{ github.event_name }}")
+ assert.equal(classifyStep.with["head-sha"], "${{ github.sha }}")
+ assert.equal(classifyStep.with["force-full-label"], step.env.FORCE_FULL_LABEL)
+ assert.equal(classifyStep.with["force-full-variable"], step.env.FORCE_FULL_VARIABLE)
+ assert.equal(gate.permissions.actions, "read")
+ assert.equal(gate.steps[0].with["fetch-depth"], 0)
+ assert.match(step.run, /node scripts\/change-gate\.mjs ci/)
+ for (const job of expectation.jobs) {
+ if (job.gated) {
+ assert.equal(jobs[job.id].needs, "classify")
+ for (const term of GATING_TERMS) assert.ok(jobs[job.id].if.includes(term))
+ assert.ok(
+ jobs[job.id].if.includes(
+ job.prOnlyCondition
+ ? "github.event_name == 'pull_request'"
+ : "github.event_name != 'pull_request'",
+ ),
+ )
+ }
+ }
+ for (const id of ["classify", "validation-gate"])
+ assert.equal(jobs[id].name, `${id} (${expectation.file.replace(/\.yml$/, "")})`)
+ })
+ }
+ it("keeps required compile and matrix checks while gating expensive steps", () => {
+ const { jobs } = parse(fs.readFileSync(path.join(repositoryRoot, ".github/workflows/code-qa.yml"), "utf8"))
+ assert.equal(jobs.compile.name, undefined)
+ assert.equal(jobs["unit-test"].name, "platform-unit-test (${{ matrix.name }})")
+ assert.deepEqual(jobs.compile.needs, ["classify", "format-check"])
+ assert.equal(jobs.compile.steps[0].env.FORMAT_RESULT, "${{ needs.format-check.result }}")
+ assert.equal(jobs.compile.steps[0].run, 'test "$FORMAT_RESULT" = success')
+ assert.ok(jobs["format-check"].steps.some((step) => step.run === "pnpm format:check"))
+ assert.ok(jobs["format-check"].steps.some((step) => step.run === "pnpm test:change-gate"))
+ for (const id of ["compile", "unit-test"]) {
+ assert.equal(jobs[id].if, "${{ !cancelled() }}")
+ assert.equal(
+ jobs[id].env.RUN_VALIDATION,
+ "${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }}",
+ )
+ for (const step of jobs[id].steps.slice(1))
+ assert.ok(step.if.includes("env.RUN_VALIDATION == 'true'"), step.name)
+ }
+ assert.equal(jobs["unit-test"].steps[0].if, "env.RUN_VALIDATION == 'false'")
+ assert.match(jobs["unit-test"].steps[0].run, /GITHUB_STEP_SUMMARY/)
+ assert.deepEqual(
+ jobs["unit-test"].strategy.matrix.include.map((entry) => entry.name),
+ ["ubuntu-latest", "windows-latest"],
+ )
+ })
+})
+
+describe("classifier stdout", () => {
+ it("parses valid output and rejects malformed output", () => {
+ assert.deepEqual(parseClassifierStdout("Change classification: full (override-label), 12 changed file(s)"), {
+ category: "full",
+ reasonCode: "override-label",
+ changedFiles: 12,
+ })
+ for (const value of [
+ undefined,
+ "",
+ "Change classification: full",
+ "Change classification: full (reason), NaN changed file(s)",
+ ])
+ assert.equal(parseClassifierStdout(value), undefined)
+ })
+})
+
+it("collects all metrics pages", () => {
+ const routes = []
+ const metrics = collectJobMetrics({
+ repository: "org/repo",
+ runId: "123",
+ fetchJson: (route) => {
+ routes.push(route)
+ if (!route.includes("/jobs?")) return { started_at: "2026-01-01T00:00:00Z" }
+ return {
+ jobs: Array.from({ length: route.endsWith("page=1") ? 100 : 1 }, (_, i) => ({
+ name: `job-${i}`,
+ started_at: "2026-01-01T00:00:00Z",
+ completed_at: "2026-01-01T00:01:00Z",
+ })),
+ }
+ },
+ })
+ assert.deepEqual(routes, [
+ "repos/org/repo/actions/runs/123",
+ "repos/org/repo/actions/runs/123/jobs?per_page=100&page=1",
+ "repos/org/repo/actions/runs/123/jobs?per_page=100&page=2",
+ ])
+ assert.equal(metrics.perJob.length, 101)
+ assert.equal(metrics.totalRunnerMinutes, 101)
+})
diff --git a/scripts/code-qa-workflow.test.mjs b/scripts/code-qa-workflow.test.mjs
index 5e1e1a59c0..0018de8fbd 100644
--- a/scripts/code-qa-workflow.test.mjs
+++ b/scripts/code-qa-workflow.test.mjs
@@ -1,4 +1,5 @@
import assert from "node:assert/strict"
+import { parse } from "yaml"
import fs from "node:fs"
import path from "node:path"
import { describe, it } from "node:test"
@@ -9,35 +10,42 @@ const workflow = fs.readFileSync(path.join(repositoryRoot, ".github/workflows/co
const extensionTurbo = JSON.parse(fs.readFileSync(path.join(repositoryRoot, "src/turbo.json"), "utf8"))
const coreTurbo = JSON.parse(fs.readFileSync(path.join(repositoryRoot, "packages/core/turbo.json"), "utf8"))
+const workflowDocument = parse(workflow)
const workflowStep = (name) => {
- const match = workflow.match(new RegExp(`- name: ${name}\\n(?(?:\\s{14,}.*\\n?)*)`))
- assert.ok(match?.groups?.body, `missing workflow step: ${name}`)
- return match.groups.body
+ const step = Object.values(workflowDocument.jobs)
+ .flatMap((job) => job.steps ?? [])
+ .find((step) => step.name === name)
+ assert.ok(step, `missing workflow step: ${name}`)
+ return step
}
-
const parseWorkflowStep = (name) => {
- const body = workflowStep(name)
- const field = (key) => {
- const line = body.split("\n").find((line) => line.trimStart().startsWith(`${key}:`))
- assert.ok(line, `missing ${key} field in workflow step: ${name}`)
- return line.slice(line.indexOf(":") + 1).trim()
- }
- return { if: field("if"), run: field("run") }
+ const step = workflowStep(name)
+ return { if: step.if, run: step.run }
}
describe("platform unit-test workflow", () => {
it("keeps coverage authoritative on Ubuntu and runs equivalent uninstrumented Windows tests", () => {
- assert.match(workflow, /name: ubuntu-latest[\s\S]*?collect-coverage: true/)
- assert.match(workflow, /name: windows-latest[\s\S]*?collect-coverage: false/)
+ assert.equal(
+ workflowDocument.jobs["unit-test"].strategy.matrix.include.find((entry) => entry.name === "ubuntu-latest")[
+ "collect-coverage"
+ ],
+ true,
+ )
+ assert.equal(
+ workflowDocument.jobs["unit-test"].strategy.matrix.include.find((entry) => entry.name === "windows-latest")[
+ "collect-coverage"
+ ],
+ false,
+ )
assert.ok(!workflow.includes("matrix.upload-coverage"))
assert.deepEqual(parseWorkflowStep("Run extension coverage lanes"), {
- if: "matrix.collect-coverage",
+ if: "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}",
run: 'pnpm turbo run test:coverage:api test:coverage:core test:coverage:services test:coverage:misc test:coverage:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only',
})
assert.deepEqual(parseWorkflowStep("Run extension test lanes"), {
- if: "${{ !matrix.collect-coverage }}",
+ if: "${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}",
run: 'pnpm turbo run test:api test:core test:services test:misc test:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only',
})
@@ -62,16 +70,16 @@ describe("platform unit-test workflow", () => {
],
]) {
const coverageStep = parseWorkflowStep(coverageStepName)
- assert.equal(coverageStep.if, "matrix.collect-coverage")
+ assert.equal(coverageStep.if, "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}")
assert.ok(coverageStep.run.includes(coverageCommand), `missing command in step: ${coverageStepName}`)
const plainStep = parseWorkflowStep(plainStepName)
- assert.equal(plainStep.if, "${{ !matrix.collect-coverage }}")
+ assert.equal(plainStep.if, "${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}")
assert.ok(plainStep.run.includes(plainCommand), `missing command in step: ${plainStepName}`)
assert.ok(!plainStep.run.includes("--coverage"), `${plainStepName} must not collect coverage on Windows`)
}
- assert.ok(!workflowStep("Run extension dist smoke test").includes("if:"))
+ assert.equal(parseWorkflowStep("Run extension dist smoke test").if, "env.RUN_VALIDATION == 'true'")
})
it("does not run coverage verification or uploads on Windows", () => {
@@ -86,7 +94,7 @@ describe("platform unit-test workflow", () => {
"Upload core integration coverage to Codecov",
"Upload coverage reports to GitHub",
]) {
- assert.match(workflowStep(stepName), /if: matrix\.collect-coverage/)
+ assert.equal(workflowStep(stepName).if, "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}")
}
})
@@ -128,23 +136,22 @@ describe("Knip quality gate", () => {
["Run knip checks", "pnpm knip"],
["Run production Knip checks", "pnpm knip:production"],
]) {
- assert.match(workflowStep(stepName), new RegExp(`run: ${command}\\s*$`))
- assert.doesNotMatch(workflowStep(stepName), /continue-on-error|if:/)
+ const step = workflowStep(stepName)
+ assert.equal(step.run, command)
+ assert.equal(step.if, undefined)
+ assert.equal(step["continue-on-error"], undefined)
}
- const job = workflow.match(/\n knip:\n(?[\s\S]*?)\n compile:/)?.groups?.body
- assert.ok(job, "missing knip job")
- assert.doesNotMatch(job, /continue-on-error|if:/)
+ assert.equal(workflowDocument.jobs.knip.if, undefined)
+ assert.equal(workflowDocument.jobs.knip["continue-on-error"], undefined)
})
})
describe("Code QA dependency install", () => {
it("installs every job with a frozen lockfile", () => {
- const setups = workflow.match(/uses: \.\/\.github\/actions\/setup-node-pnpm\n/g) ?? []
- const frozen =
- workflow.match(
- /uses: \.\/\.github\/actions\/setup-node-pnpm\n\s+with:\n\s+install-args: "--frozen-lockfile"\n/g,
- ) ?? []
+ const setups = Object.values(workflowDocument.jobs)
+ .flatMap((job) => job.steps ?? [])
+ .filter((step) => step.uses === "./.github/actions/setup-node-pnpm")
assert.ok(setups.length > 0, "missing setup-node-pnpm steps")
- assert.equal(frozen.length, setups.length)
+ for (const step of setups) assert.equal(step.with["install-args"], "--frozen-lockfile")
})
})
diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs
index 339e44e676..abf96eab65 100644
--- a/scripts/stryker-diff.test.mjs
+++ b/scripts/stryker-diff.test.mjs
@@ -81,9 +81,15 @@ describe("mutation testing workflow", () => {
it("waits until a draft pull request is ready before emitting mutation annotations", () => {
const workflow = readWorkflow()
- assert.ok(workflow.includes("types: [edited, opened, reopened, ready_for_review, synchronize]"))
+ assert.ok(parseYaml(workflow).on.pull_request.types.includes("ready_for_review"))
+ // The job condition keeps the draft and merge-group semantics asserted
+ // below and adds the fail-safe change-classification gate from issue
+ // #2291: only a successful documentation-only pull-request
+ // classification may skip the job.
assert.ok(
- workflow.includes("if: github.event_name == 'merge_group' || github.event.pull_request.draft == false"),
+ workflow.includes(
+ "if: ${{ !cancelled() && (github.event_name == 'merge_group' || github.event.pull_request.draft == false) && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}",
+ ),
)
const draftToReadyRuns = [