diff --git a/.github/actions/classify-changes/action.yml b/.github/actions/classify-changes/action.yml new file mode 100644 index 0000000000..cad30a10c7 --- /dev/null +++ b/.github/actions/classify-changes/action.yml @@ -0,0 +1,56 @@ +name: "Classify changed files" + +description: "Classifies a pull request change range as documentation-only or full validation. The classification fails safe: every uncertain input selects full validation." + +inputs: + event-name: + description: "GitHub event name that drives skip eligibility" + required: true + head-sha: + description: "Head commit SHA of the classification range" + required: true + force-full-label: + description: "When true, force full validation because the ci:full-validation label is present" + required: false + default: "false" + force-full-variable: + description: "When true, force full validation because the ZOO_CODE_REQUIRE_FULL_VALIDATION repository variable is true" + required: false + default: "false" + +outputs: + category: + description: "docs-only or full" + value: ${{ steps.classify.outputs.category }} + reason: + description: "Human-readable classification reason" + value: ${{ steps.classify.outputs.reason }} + reason-code: + description: "Stable classification reason code" + value: ${{ steps.classify.outputs.reason-code }} + changed-files: + description: "Number of changed files in the classification range" + value: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: + description: "Fingerprint of the active documentation-only allowlist" + value: ${{ steps.classify.outputs.allowlist-fingerprint }} + +runs: + using: "composite" + steps: + - id: classify + shell: bash + run: | + HEAD_SHA="${{ inputs.head-sha }}" + FLAGS=() + if [[ "${{ inputs.force-full-label }}" == "true" ]]; then + FLAGS+=("--force-full-label") + fi + if [[ "${{ inputs.force-full-variable }}" == "true" ]]; then + FLAGS+=("--force-full-variable") + fi + # The pull_request checkout is the synthetic merge commit, so its + # first parent is the base branch tip that was actually merged + # into. A failed resolution fails the step and fails safe. + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-classification.mjs ci --event "${{ inputs.event-name }}" --head "$HEAD_SHA" --base "$BASE_SHA" "${FLAGS[@]}" diff --git a/.github/workflows/code-qa.yml b/.github/workflows/code-qa.yml index 42b94efa3d..35287a98eb 100644 --- a/.github/workflows/code-qa.yml +++ b/.github/workflows/code-qa.yml @@ -5,7 +5,7 @@ on: push: branches: [main] pull_request: - types: [opened, reopened, ready_for_review, synchronize] + types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled] merge_group: types: [checks_requested] @@ -20,6 +20,37 @@ permissions: contents: read jobs: + classify: + name: classify (code-qa) + # Classifies the pull request change range before any validation + # runs. Only an explicit documentation-only allowlist match can + # skip expensive jobs, and only for pull_request events; a failed + # or missing classification fails safe to full validation. The + # validation-gate job below audits this decision on every run. + runs-on: ubuntu-latest + outputs: + category: ${{ steps.classify.outputs.category }} + reason: ${{ steps.classify.outputs.reason }} + reason-code: ${{ steps.classify.outputs.reason-code }} + changed-files: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Classification resolves the merge base from history. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Classify changed files + id: classify + uses: ./.github/actions/classify-changes + with: + event-name: ${{ github.event_name }} + head-sha: ${{ github.sha }} + force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + dependency-review: runs-on: ubuntu-latest # Only meaningful for PRs — validates the dependency diff of the pull @@ -65,7 +96,30 @@ jobs: exit 1 fi + format-check: + name: Formatting and validation policy tests + # Always runs, including documentation-only pull requests: Prettier + # checks every Markdown file, so formatting validation must never be + # reduced by classification. This job also validates the classifier + # and gate logic on every run. + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Node.js and pnpm + uses: ./.github/actions/setup-node-pnpm + with: + install-args: "--frozen-lockfile" + - name: Check formatting + run: pnpm format:check + - name: Test change classification and validation gate + run: pnpm test:change-gate + check-translations: + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Everything else runs unchanged. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest steps: - name: Checkout code @@ -80,6 +134,8 @@ jobs: run: node scripts/check-i18n-interpolation.js knip: + # Intentionally ungated: the repository invariant test requires the + # knip job to run without conditions or allowed failure. runs-on: ubuntu-latest steps: - name: Checkout code @@ -96,42 +152,63 @@ jobs: run: pnpm knip:production compile: + needs: [classify, format-check] + # Start the required check even for docs-only changes. + if: ${{ !cancelled() }} + env: + RUN_VALIDATION: ${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }} runs-on: ubuntu-latest steps: + - name: Enforce formatting and validation policy tests + env: + FORMAT_RESULT: ${{ needs.format-check.result }} + run: test "$FORMAT_RESULT" = success - name: Checkout code + if: env.RUN_VALIDATION == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js and pnpm + if: env.RUN_VALIDATION == 'true' uses: ./.github/actions/setup-node-pnpm with: install-args: "--frozen-lockfile" - - name: Check formatting - run: pnpm format:check - name: Lint + if: env.RUN_VALIDATION == 'true' run: pnpm lint - name: Check types + if: env.RUN_VALIDATION == 'true' run: pnpm check-types - name: Validate Code QA workflow + if: env.RUN_VALIDATION == 'true' run: pnpm test:code-qa-ci - name: Validate Solheim provider smoke workflow + if: env.RUN_VALIDATION == 'true' run: pnpm test:solheim-smoke-ci - name: Test Solheim provider smoke units + if: env.RUN_VALIDATION == 'true' run: pnpm test:solheim-smoke:unit - name: Check Solheim provider smoke types + if: env.RUN_VALIDATION == 'true' run: pnpm solheim-smoke:check-types - name: Test agent evals units + if: env.RUN_VALIDATION == 'true' run: pnpm test:agent-evals:unit - name: Check agent evals types + if: env.RUN_VALIDATION == 'true' run: pnpm agent-evals:check-types - name: Model-check task lifecycle protocols + if: env.RUN_VALIDATION == 'true' run: pnpm lifecycle:model-check - name: Model-check webview recovery + if: env.RUN_VALIDATION == 'true' run: pnpm webview-recovery:check-types && pnpm webview-recovery:model-check - name: Validate MCP OAuth integration + if: env.RUN_VALIDATION == 'true' run: pnpm mcp:integration-check build-vsix: name: Build test VSIX - if: github.event_name == 'pull_request' + needs: classify + if: ${{ !cancelled() && github.event_name == 'pull_request' && (needs.classify.result != 'success' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest steps: - name: Checkout code @@ -158,6 +235,11 @@ jobs: unit-test: name: platform-unit-test (${{ matrix.name }}) + needs: classify + # Start the required check even for docs-only changes. + if: ${{ !cancelled() }} + env: + RUN_VALIDATION: ${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }} runs-on: ${{ matrix.os }} concurrency: # Duplicate runs for one commit must not race the OS-specific Turbo cache save. @@ -175,13 +257,20 @@ jobs: codecov-flag: windows collect-coverage: false steps: + - name: Report documentation-only validation + if: env.RUN_VALIDATION == 'false' + shell: bash + run: echo "Documentation-only change; expensive platform validation steps skipped." >> "$GITHUB_STEP_SUMMARY" - name: Checkout code + if: env.RUN_VALIDATION == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node.js and pnpm + if: env.RUN_VALIDATION == 'true' uses: ./.github/actions/setup-node-pnpm with: install-args: "--frozen-lockfile" - name: Restore Turbo cache + if: env.RUN_VALIDATION == 'true' id: turbo-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -194,16 +283,16 @@ jobs: # suites through the uninstrumented Turbo tasks. Ubuntu stays the # authoritative coverage lane. - name: Run non-extension package coverage - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm turbo run test:coverage --filter="!@roo-code/core" --filter="!zoo-code" --log-order grouped --output-logs new-only - name: Run non-extension package tests - if: ${{ !matrix.collect-coverage }} + if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }} run: pnpm turbo run test --filter="!@roo-code/core" --filter="!zoo-code" --log-order grouped --output-logs new-only - name: Run extension coverage lanes - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm turbo run test:coverage:api test:coverage:core test:coverage:services test:coverage:misc test:coverage:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only - name: Run extension test lanes - if: ${{ !matrix.collect-coverage }} + if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }} run: pnpm turbo run test:api test:core test:services test:misc test:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only # Runs outside Turbo so changes to .coderabbit.yaml, # .github/workflows/label-pr-review-state.yml, and @@ -211,26 +300,28 @@ jobs: # sit outside the src/ package boundary and cannot be declared as # Turbo inputs. - name: Run workflow config tests + if: env.RUN_VALIDATION == 'true' run: pnpm --dir src run test:workflow - name: Verify extension coverage contract - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm --dir src run verify:coverage-contract - name: Run extension dist smoke test + if: env.RUN_VALIDATION == 'true' run: pnpm turbo run test:dist --filter="zoo-code" --log-order grouped --output-logs new-only - name: Run core unit coverage - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm turbo run test:coverage:unit --filter="@roo-code/core" --log-order grouped --output-logs new-only - name: Run core unit tests - if: ${{ !matrix.collect-coverage }} + if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }} run: pnpm turbo run test:unit --filter="@roo-code/core" --log-order grouped --output-logs new-only - name: Run core integration coverage - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm turbo run test:coverage:integration --filter="@roo-code/core" --log-order grouped --output-logs new-only - name: Run core integration tests - if: ${{ !matrix.collect-coverage }} + if: ${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }} run: pnpm turbo run test:integration --filter="@roo-code/core" --log-order grouped --output-logs new-only - name: Verify extension coverage reports - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: | node src/scripts/verify-lcov.mjs src/coverage/api/lcov.info node src/scripts/verify-lcov.mjs src/coverage/core/lcov.info @@ -238,17 +329,17 @@ jobs: node src/scripts/verify-lcov.mjs src/coverage/misc/lcov.info node src/scripts/verify-lcov.mjs src/coverage/tree-sitter/lcov.info - name: Merge extension coverage reports - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: | mkdir -p src/coverage/merged pnpm --dir src run merge:coverage node src/scripts/verify-lcov.mjs src/coverage/merged/lcov.info # Validate cache boundaries before publishing any new Turbo entries. - name: Verify coverage cache inputs - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} run: pnpm --dir src run verify:coverage-cache-inputs - name: Save Turbo cache - if: steps.turbo-cache.outputs.cache-hit != 'true' + if: ${{ env.RUN_VALIDATION == 'true' && (steps.turbo-cache.outputs.cache-hit != 'true') }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .turbo/cache @@ -263,7 +354,7 @@ jobs: # it. Core and webview reports retain their independent flags. # See https://docs.codecov.com/docs/flags - name: Upload non-core coverage to Codecov - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: >- @@ -275,7 +366,7 @@ jobs: flags: ${{ matrix.codecov-flag }} token: ${{ secrets.CODECOV_TOKEN }} - name: Upload webview JSDOM coverage to Codecov - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: webview-ui/coverage/lcov.info @@ -283,7 +374,7 @@ jobs: flags: webview-ui token: ${{ secrets.CODECOV_TOKEN }} - name: Upload core unit coverage to Codecov - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: packages/core/coverage/unit/lcov.info @@ -291,7 +382,7 @@ jobs: flags: ${{ matrix.codecov-flag }},core-unit token: ${{ secrets.CODECOV_TOKEN }} - name: Upload core integration coverage to Codecov - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: files: packages/core/coverage/integration/lcov.info @@ -299,7 +390,7 @@ jobs: flags: ${{ matrix.codecov-flag }},core-integration token: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage reports to GitHub - if: matrix.collect-coverage + if: ${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-reports-${{ matrix.name }} @@ -317,3 +408,50 @@ jobs: packages/core/coverage/unit/lcov.info packages/core/coverage/integration/lcov.info retention-days: 7 + + validation-gate: + name: validation-gate (code-qa) + # Always-running audit gate for change-based validation. It + # rejects a missing or failed classification, validation jobs that + # did not run when full validation was required, jobs that ran when + # a documentation-only skip was expected, and a classifier replay + # that disagrees with the recorded classification. It also records + # the classification, the skipped work, runner minutes, and the + # required-check completion time in the job summary. + if: always() + needs: + - classify + - dependency-review + - invisible-chars + - format-check + - check-translations + - knip + - compile + - build-vsix + - unit-test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The gate replays the classifier against the same range. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Run validation gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_JSON: ${{ toJSON(needs) }} + EVENT_NAME: ${{ github.event_name }} + PR_DRAFT: ${{ github.event.pull_request.draft }} + RUN_ID: ${{ github.run_id }} + HEAD_SHA: ${{ github.sha }} + FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + WATCHED_JOBS: dependency-review:pr-only:ungated,invisible-chars:always:ungated,format-check:always:ungated,check-translations:always:gated,knip:always:ungated,compile:always:ungated,build-vsix:pr-only:gated,unit-test:always:ungated + run: | + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 27bfd3dec1..4d1d294720 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -4,6 +4,7 @@ on: push: branches: ["main"] pull_request: + types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled] schedule: - cron: "24 19 * * 3" @@ -19,8 +20,44 @@ permissions: contents: read jobs: + classify: + name: classify (codeql) + # Classifies the pull request change range before any validation + # runs. Only an explicit documentation-only allowlist match can + # skip expensive jobs, and only for pull_request events; a failed + # or missing classification fails safe to full validation. The + # validation-gate job below audits this decision on every run. + runs-on: ubuntu-latest + outputs: + category: ${{ steps.classify.outputs.category }} + reason: ${{ steps.classify.outputs.reason }} + reason-code: ${{ steps.classify.outputs.reason-code }} + changed-files: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Classification resolves the merge base from history. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Classify changed files + id: classify + uses: ./.github/actions/classify-changes + with: + event-name: ${{ github.event_name }} + head-sha: ${{ github.sha }} + force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + analyze: name: Analyze (${{ matrix.language }}) + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue, push, and scheduled + # analysis always runs. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources @@ -89,3 +126,43 @@ jobs: uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: category: "/language:${{matrix.language}}" + + validation-gate: + name: validation-gate (codeql) + # Always-running required gate for change-based validation. It + # rejects a missing or failed classification, validation jobs that + # did not run when full validation was required, jobs that ran when + # a documentation-only skip was expected, and a classifier replay + # that disagrees with the recorded classification. It also records + # the classification, the skipped work, runner minutes, and the + # required-check completion time in the job summary. + if: always() + needs: + - classify + - analyze + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The gate replays the classifier against the same range. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Run validation gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_JSON: ${{ toJSON(needs) }} + EVENT_NAME: ${{ github.event_name }} + PR_DRAFT: ${{ github.event.pull_request.draft }} + RUN_ID: ${{ github.run_id }} + HEAD_SHA: ${{ github.sha }} + FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + WATCHED_JOBS: analyze:always:gated + run: | + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9c86fe9f77..0c20bcaaf3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,7 +3,7 @@ name: E2E Tests (Mocked) on: workflow_dispatch: pull_request: - types: [opened, reopened, ready_for_review, synchronize] + types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled] merge_group: types: [checks_requested] @@ -17,7 +17,42 @@ permissions: contents: read jobs: + classify: + name: classify (e2e) + # Classifies the pull request change range before any validation + # runs. Only an explicit documentation-only allowlist match can + # skip expensive jobs, and only for pull_request events; a failed + # or missing classification fails safe to full validation. The + # validation-gate job below audits this decision on every run. + runs-on: ubuntu-latest + outputs: + category: ${{ steps.classify.outputs.category }} + reason: ${{ steps.classify.outputs.reason }} + reason-code: ${{ steps.classify.outputs.reason-code }} + changed-files: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Classification resolves the merge base from history. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Classify changed files + id: classify + uses: ./.github/actions/classify-changes + with: + event-name: ${{ github.event_name }} + head-sha: ${{ github.sha }} + force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + e2e-mock: + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue validation always runs. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -191,3 +226,43 @@ jobs: with: path: .cache/e2e-pass key: ${{ steps.e2e-marker.outputs.cache-primary-key }} + + validation-gate: + name: validation-gate (e2e) + # Always-running required gate for change-based validation. It + # rejects a missing or failed classification, validation jobs that + # did not run when full validation was required, jobs that ran when + # a documentation-only skip was expected, and a classifier replay + # that disagrees with the recorded classification. It also records + # the classification, the skipped work, runner minutes, and the + # required-check completion time in the job summary. + if: always() + needs: + - classify + - e2e-mock + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The gate replays the classifier against the same range. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Run validation gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_JSON: ${{ toJSON(needs) }} + EVENT_NAME: ${{ github.event_name }} + PR_DRAFT: ${{ github.event.pull_request.draft }} + RUN_ID: ${{ github.run_id }} + HEAD_SHA: ${{ github.sha }} + FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + WATCHED_JOBS: e2e-mock:always:gated + run: | + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml index fd2a00747a..38aedaa8a1 100644 --- a/.github/workflows/mutation-testing.yml +++ b/.github/workflows/mutation-testing.yml @@ -2,7 +2,7 @@ name: Changed-code mutation testing on: pull_request: - types: [edited, opened, reopened, ready_for_review, synchronize] + types: [edited, opened, reopened, ready_for_review, synchronize, labeled, unlabeled] merge_group: types: [checks_requested] @@ -14,9 +14,43 @@ concurrency: cancel-in-progress: true jobs: + classify: + name: classify (mutation-testing) + # Classifies the pull request change range before any validation + # runs. Only an explicit documentation-only allowlist match can + # skip expensive jobs, and only for pull_request events; a failed + # or missing classification fails safe to full validation. The + # validation-gate job below audits this decision on every run. + runs-on: ubuntu-latest + outputs: + category: ${{ steps.classify.outputs.category }} + reason: ${{ steps.classify.outputs.reason }} + reason-code: ${{ steps.classify.outputs.reason-code }} + changed-files: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Classification resolves the merge base from history. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Classify changed files + id: classify + uses: ./.github/actions/classify-changes + with: + event-name: ${{ github.event_name }} + head-sha: ${{ github.sha }} + force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + mutation-diff: name: mutation-diff - if: github.event_name == 'merge_group' || github.event.pull_request.draft == false + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue validation always runs. + if: ${{ !cancelled() && (github.event_name == 'merge_group' || github.event.pull_request.draft == false) && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -76,3 +110,43 @@ jobs: echo "### Download mutation reports" echo "[Open the changed-code-mutation-report artifact]($ARTIFACT_URL), then open the package's mutation.html file." } >> "$GITHUB_STEP_SUMMARY" || echo "::warning title=Mutation test advisory::Could not write the job summary" + + validation-gate: + name: validation-gate (mutation-testing) + # Always-running required gate for change-based validation. It + # rejects a missing or failed classification, validation jobs that + # did not run when full validation was required, jobs that ran when + # a documentation-only skip was expected, and a classifier replay + # that disagrees with the recorded classification. It also records + # the classification, the skipped work, runner minutes, and the + # required-check completion time in the job summary. + if: always() + needs: + - classify + - mutation-diff + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The gate replays the classifier against the same range. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Run validation gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_JSON: ${{ toJSON(needs) }} + EVENT_NAME: ${{ github.event_name }} + PR_DRAFT: ${{ github.event.pull_request.draft }} + RUN_ID: ${{ github.run_id }} + HEAD_SHA: ${{ github.sha }} + FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + WATCHED_JOBS: mutation-diff:pr-non-draft:gated + run: | + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml index a3e7bb939b..744d55d162 100644 --- a/.github/workflows/visual-regression.yml +++ b/.github/workflows/visual-regression.yml @@ -7,7 +7,7 @@ on: # that do not match the filter, which blocks merging for a required check. # The merge_group event ignores path filters, so the queue is unaffected. pull_request: - types: [opened, reopened, ready_for_review, synchronize] + types: [opened, reopened, ready_for_review, synchronize, labeled, unlabeled] merge_group: types: [checks_requested] @@ -19,7 +19,42 @@ permissions: contents: read jobs: + classify: + name: classify (visual-regression) + # Classifies the pull request change range before any validation + # runs. Only an explicit documentation-only allowlist match can + # skip expensive jobs, and only for pull_request events; a failed + # or missing classification fails safe to full validation. The + # validation-gate job below audits this decision on every run. + runs-on: ubuntu-latest + outputs: + category: ${{ steps.classify.outputs.category }} + reason: ${{ steps.classify.outputs.reason }} + reason-code: ${{ steps.classify.outputs.reason-code }} + changed-files: ${{ steps.classify.outputs.changed-files }} + allowlist-fingerprint: ${{ steps.classify.outputs.allowlist-fingerprint }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Classification resolves the merge base from history. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Classify changed files + id: classify + uses: ./.github/actions/classify-changes + with: + event-name: ${{ github.event_name }} + head-sha: ${{ github.sha }} + force-full-label: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + force-full-variable: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + webview-visual: + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue validation always runs. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest timeout-minutes: 10 container: @@ -55,6 +90,10 @@ jobs: if-no-files-found: ignore theme-fixtures: + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue validation always runs. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -84,6 +123,10 @@ jobs: run: xvfb-run -a pnpm --filter @roo-code/vscode-e2e themes:check extension-host-visual: + needs: classify + # Fail-safe: only a successful documentation-only classification on + # a pull request skips this job. Merge-queue validation always runs. + if: ${{ !cancelled() && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }} runs-on: ubuntu-latest # Includes dependency setup, builds, repeated visuals, and production watchdog waits. timeout-minutes: 35 @@ -124,3 +167,45 @@ jobs: apps/vscode-e2e/playwright-report-electron apps/vscode-e2e/test-results-electron if-no-files-found: ignore + + validation-gate: + name: validation-gate (visual-regression) + # Always-running required gate for change-based validation. It + # rejects a missing or failed classification, validation jobs that + # did not run when full validation was required, jobs that ran when + # a documentation-only skip was expected, and a classifier replay + # that disagrees with the recorded classification. It also records + # the classification, the skipped work, runner minutes, and the + # required-check completion time in the job summary. + if: always() + needs: + - classify + - webview-visual + - theme-fixtures + - extension-host-visual + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The gate replays the classifier against the same range. + fetch-depth: 0 + # This job never pushes — don't persist the GITHUB_TOKEN. + persist-credentials: false + - name: Run validation gate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEEDS_JSON: ${{ toJSON(needs) }} + EVENT_NAME: ${{ github.event_name }} + PR_DRAFT: ${{ github.event.pull_request.draft }} + RUN_ID: ${{ github.run_id }} + HEAD_SHA: ${{ github.sha }} + FORCE_FULL_LABEL: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:full-validation') }} + FORCE_FULL_VARIABLE: ${{ vars.ZOO_CODE_REQUIRE_FULL_VALIDATION == 'true' }} + WATCHED_JOBS: webview-visual:always:gated,theme-fixtures:always:gated,extension-host-visual:always:gated + run: | + BASE_SHA="$(git rev-parse "${HEAD_SHA}^1")" + node scripts/change-gate.mjs ci --base "$BASE_SHA" --head "$HEAD_SHA" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61bb417bf2..38e7032c02 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,8 @@ Zoo Code is a community project. We value every contribution. We work on an [Iss - [Writing Style](#writing-style) - [Legal](#legal) +See [change-based CI validation](docs/ci-change-based-validation.md) for validation skips and full-validation overrides. + ## Before You Contribute ### 1. Code of Conduct diff --git a/docs/ci-change-based-validation.md b/docs/ci-change-based-validation.md new file mode 100644 index 0000000000..3a2763752f --- /dev/null +++ b/docs/ci-change-based-validation.md @@ -0,0 +1,226 @@ +# Change-based validation with an audit gate + +Every pull request update previously ran the full validation suite even when the +change could not affect product behavior. This document describes the +conservative change-based validation introduced for issue #2291, its safety +model, its override and rollback paths, and the evidence collected before +skips were enabled. + +## How it works + +Five workflows (`code-qa`, `e2e`, `visual-regression`, `mutation-testing`, +`codeql`) run a fast `classify` job first. It runs +`scripts/change-classification.mjs` over the pull request's full change range +and emits either `docs-only` or `full`. Expensive jobs read that result through +their `needs` graph: + +- A job is skipped only when classification succeeded, the event is + `pull_request`, and the category is exactly `docs-only`. The job-level + condition reports `skipped`, and GitHub treats a skipped required check as + successful. The required unit-test matrix jobs start both platform checks + and gate their steps instead, because GitHub evaluates job conditions before + expanding a matrix. Skipping the matrix job itself may leave its named + required contexts unreported. + The required `compile` job also starts every time and fails unless the + always-running formatting and validation-policy tests passed; only its + expensive steps are gated. This preserves the existing check names without a ruleset change; live + verification is still pending. +- Every other outcome runs the job unchanged: category `full`, a failed + classification, a missing classification, or any non-`pull_request` event + (merge queue, pushes, scheduled runs). + +Each workflow also runs a `validation-gate` job that always executes. +The classify and gate checks have unique display names, such as +`classify (code-qa)` and `validation-gate (code-qa)`, for branch rulesets. The gate +replays the classifier against the same commit range and rejects the run when: + +- the classification is missing, malformed, or non-terminal, +- a validation job did not run when full validation was required, +- a validation job ran when a documentation-only skip was expected, +- the replay disagrees with the recorded classification, +- a watched job is missing from the `needs` graph. + +On merge-queue events classification never reduces validation; existing +event and draft conditions still apply, including skips for PR-only jobs. +CodeQL does not currently subscribe to merge-queue events. The gate writes a job +summary with the classification, the expected and actual outcome of every +watched job, best-effort runner minutes per job, the total runner minutes of the run, and +the time from run start to the last completed job (the required-check +completion time). + +## The documentation-only allowlist + +`scripts/change-classification.mjs` holds one explicit allowlist. Only these +files can produce a `docs-only` classification: + +| Pattern | Reason | +| -------------------- | ------------------------------------------------- | +| `CONTRIBUTING.md` | Contributor guide. | +| `SECURITY.md` | Security policy. | +| `PRIVACY.md` | Privacy notice. | +| `CODE_OF_CONDUCT.md` | Code of conduct. | +| `docs/**/*.md` | Documentation tree; contains only Markdown today. | +| `.github/**/*.md` | Issue and pull request templates. | + +The allowlist is deliberately narrow: + +- Release inputs stay out. `CHANGELOG.md`, `src/CHANGELOG.md`, `locales/*/README.md`, + `README.md` (copied by `src/esbuild.mjs` into the packaged extension), + and packaging manifests trigger full validation, and the separate + release-validation workflow is untouched. +- Workflow and toolchain configuration stays out. Only `.md` files under + `.github/` match; executable workflow/action YAML and `CODEOWNERS` select + full validation. Inert Markdown notes under workflows or actions may skip. +- Embedded fixtures, executable examples, and generated inputs never skip: + any path containing `fixtures`, `__tests__`, `__mocks__`, `test-utils`, + `snapshots`, `__snapshots__`, `generated`, `examples`, or `testdata` is + rejected even under an allowlisted prefix. +- The `format-check` job always runs because `pnpm format:check` validates + every Markdown file; documentation changes keep their own check. +- The `knip` and `invisible-chars` jobs always run. `dependency-review` is + never gated by classification but stays pull-request-only. Knip is + deliberately unconditional because a repository invariant test requires + the knip job to run without conditions or allowed failure. + +## Documentation input audit + +The audit at `687f8489d` checked references to the allowlisted paths and their +file names across tracked source, tests, build scripts, and CI configuration. +It also checked package build entry points, test discovery, asset copying, +Turbo inputs, and VSIX packaging rules. + +| Surface | Evidence | Result | +| -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Extension bundling | `src/esbuild.mjs` calls `copyPaths()` with an explicit source list. `packages/build/src/esbuild.ts` copies only those configured paths. | Root `README.md` and `CHANGELOG.md` are package inputs and remain outside the allowlist. No allowlisted documentation is copied. | +| VSIX packaging | `src/.vscodeignore` excludes everything, then includes the built extension, webview, assets, and release metadata. | Root `docs/`, `.github/`, and the four allowlisted root files are not packaged. | +| Webview and package builds | `webview-ui/vite.config.ts`, `apps/cli/tsup.config.ts`, and `packages/types/tsup.config.ts` select code entry points and assets. | No imports, raw-file loads, or asset rules select allowlisted documentation. | +| Tests and models | Vitest configurations select code tests. Searches covered `docs/`, current documentation basenames, Markdown imports, and filesystem readers. | Documentation references are comments, temporary repository files, mocked paths, or workspace examples. Tests do not read the checked-in allowlisted documents. | +| Turbo and Knip | Root and package Turbo files define package-local inputs. `knip.json` selects code entry points and projects. | No explicit input reaches the root documentation tree. Default package inputs can include package-local Markdown, which remains outside the allowlist. | +| Release validation | `.github/workflows/release-validation.yml` builds the extension and compares root `README.md` with the copied package README. | Release inputs retain full validation. The release workflow is unchanged. | + +Generic runtime tools can read Markdown in a user's workspace. This does not +make repository documentation a build or test input. The audit found no reason +to narrow the current allowlist further. This is source-level evidence for the +current tree, not a guarantee about future consumers. A new documentation +reader must update the classification policy and its regression tests. + +## Fail-safe behavior + +| Input | Classification | +| --------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| All changed files match the allowlist (additions, modifications, deletions, documentation-to-documentation renames, type changes) | `docs-only` | +| Any path outside the allowlist, including source, dependency manifests, lockfiles, workflow files | `full` | +| Rename from or into a non-allowlisted path (both sides are classified) | `full` | +| Empty change range | `full` | +| Change status the classifier does not recognize (for example copies) | `full` | +| Missing or unresolvable merge base, shallow history, unknown commits | Classification fails; jobs fall back to full validation; the gate records the fallback | +| Any git error or malformed diff output | Classification fails; jobs fall back to full validation | + +The changed-file enumeration uses NUL-delimited `git diff --name-status` over +the full merge-base range with rename detection, so it cannot silently +truncate on whitespace or large diffs. Rename entries classify the old and the +new path. + +Changes to `scripts/change-classification.mjs` or `scripts/change-gate.mjs` +always select full validation, independently of the documentation allowlist. +The gate also checks these paths directly with git against the base range +before accepting a documentation-only result. An allowlist expansion cannot +classify its own policy change as documentation-only. The fingerprint remains +an observability value rather than a trust boundary. + +## Override and rollback + +Full validation can be forced at three levels: + +1. **Per pull request:** add the `ci:full-validation` label. The classify job + reports `full` with reason `override-label`. +2. **Repository-wide:** set the repository variable + `ZOO_CODE_REQUIRE_FULL_VALIDATION` to `true`. No code change is needed; + every classification reports `full` with reason `override-variable`. +3. **Permanent:** revert the pull request that introduced this change. All + workflows then run exactly as before. + +The label must exist in the repository before it can be applied. Adding or +removing a label starts a fresh run in all five workflows. Their concurrency +groups replace superseded PR runs, so the latest label state determines the +active classification. Repository variables do not trigger runs: push an update +or dispatch a fresh run after changing the variable. Re-running an old run +retains its original event payload. + +The gate applies the same override rules when it replays the classifier, so an +overridden run still validates coherence. + +## Historical validation before enabling skips + +The original pre-enablement review snapshot (PR #2325 at `d13547de0`) +reported the following results over the 500 most recent +first-parent commits of `main` with `pull_request` semantics +(`node scripts/change-classification.mjs history --limit 500`). These are +historical results, not a current measurement; the original capture date was +not recorded. To reproduce the sample, fetch +complete history first (`git fetch --unshallow` for a shallow checkout), check +out the snapshot, and record the date and main SHA with the report: + +- 499 commits classified `full` (reason `non-documentation-path`). +- 1 commit classified `docs-only`: `a704002b67fd`, "[Docs] Specify P1 + persisted ownership and add the run-state kernel target architecture + (#1706)". Its eight files are all `docs/architecture/*.md`; skipping + comprehensive validation for that change is correct. +- 0 classification failures and 0 unrecognized change statuses: the parser + handled every real change set in the sample, including renames and + deletions. + +The sample starts narrow by design: about 0.2% of changes would have skipped, +and the only proposed skip was verified file by file. Skipped work is always +explained in the gate summary. + +## Expanding the allowlist + +Expand the allowlist only with dependency evidence: + +1. Prove the new pattern is not a build, runtime, or test input: grep the + test suites, build scripts, Knip configuration, and Turbo inputs for reads + of the path. +2. Extend `scripts/change-classification.test.mjs` with accept and reject + cases for the new pattern. +3. Run the history mode over a large sample and verify every newly proposed + skip by hand. +4. Expand one pattern at a time and watch the gate summaries for unexpected + skips. + +Broader package selection (running only affected packages) remains out of +scope until the shared-input audit is done; Zoo imports shared source outside +package-local directories. Existing coverage slicing (issue #47) is a separate +effort. A comprehensive merge-queue run is additional protection, not a +substitute for correct pull request classification. + +## Trust boundary and maintenance + +Replay detects accidental disagreement. On `pull_request`, workflow, action, +classifier, and gate code all come from the PR merge commit; the gate is not +independent protection against a PR author who edits validation itself. +Maintainers should review `.github/**` and `scripts/change-*.mjs` as CI policy; +CODEOWNERS protection for these paths is a possible separate policy change. +The gate jobs must be added to branch rulesets if maintainers want gate failures +themselves to block merging. Formatting and policy-test failures already flow +through the existing required `compile` context. + +The repeated gate jobs remain explicit to keep workflow needs and check names +visible. Extracting a shared gate action is deferred to a separate refactor. + +## Current verification and open work + +No live docs-only PR run has confirmed the two platform check contexts yet. +Local workflow tests verify their names and wiring; this does not establish +GitHub's live check-reporting behavior. + +`format-check` and all `validation-gate` jobs are not required checks in the +current ruleset. Formatting and policy-test failures block via required +`compile`. Making audit-gate failures block directly still requires a separate +ruleset update. The gate reads job results and cannot observe step-level gating +inside `compile` or `unit-test`; workflow tests pin those step conditions. + +All label changes currently start all five workflows and may cancel in-flight +PR validation, including mutation testing. This runner-time cost is accepted +for now to keep override event handling simple and consistent. A shared gate +action and broader Stryker integration remain follow-ups. diff --git a/package.json b/package.json index a67f824dcc..a8bf97f0e9 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", + "test:change-gate": "node --test scripts/change-classification.test.mjs scripts/change-gate.test.mjs", "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-file-replacement-restart.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check", "webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts", "webview-recovery:check-types": "tsc -p scripts/tsconfig.webview-recovery.json", diff --git a/scripts/change-classification.mjs b/scripts/change-classification.mjs new file mode 100644 index 0000000000..40b18854f3 --- /dev/null +++ b/scripts/change-classification.mjs @@ -0,0 +1,440 @@ +#!/usr/bin/env node + +// Conservative change-based validation classifier (issue #2291). +// +// Classifies the full changed-file range of a pull request as +// "docs-only" (safe to skip expensive comprehensive validation) or +// "full" (run everything). The classification is fail-safe: every +// uncertain input — unknown paths, unrecognized change statuses, +// missing history, malformed git output, empty ranges — selects full +// validation. Only an explicit allowlist of documentation-only files +// whose contents are not build, runtime, or test inputs can produce +// "docs-only", and only for pull_request events. + +import { execFileSync } from "node:child_process" +import { createHash } from "node:crypto" +import fs from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" + +// Exact repository files that are pure documentation. Enumerated +// individually so an unexpected new root-level markdown file (for +// example a release-affecting CHANGELOG.md) never matches by accident. +const DOCS_ONLY_FILES = [ + { path: "CONTRIBUTING.md", description: "contributor guide" }, + { path: "SECURITY.md", description: "security policy" }, + { path: "PRIVACY.md", description: "privacy notice" }, + { path: "CODE_OF_CONDUCT.md", description: "code of conduct" }, +] + +// Directories where only Markdown files qualify. Everything else under +// these prefixes (images, configs, scripts) still selects full +// validation. +const DOCS_ONLY_MARKDOWN_PREFIXES = [ + { prefix: "docs/", description: "documentation tree" }, + { prefix: ".github/", description: "repository templates" }, +] + +// Defense in depth: these path segments mark files that may be loaded +// by builds, tests, or examples even when they sit under an allowlisted +// prefix. A Markdown file inside a fixtures or examples directory is a +// build/runtime/test input, so it must never allow a skip. +export const NEVER_SKIP_SEGMENTS = [ + "fixtures", + "__tests__", + "__mocks__", + "test-utils", + "snapshots", + "__snapshots__", + "generated", + "examples", + "testdata", +] + +// Policy paths stay protected even if the documentation allowlist expands. +const VALIDATION_POLICY_PATHS = new Set(["scripts/change-classification.mjs", "scripts/change-gate.mjs"]) + +// Statuses understood from `git diff --name-status`. Any other letter +// (including C for copies and X for unknown) is unrecognized and fails +// safe to full validation. +const KNOWN_STATUSES = new Set(["A", "M", "D", "R", "T"]) + +export const SKIP_ELIGIBLE_EVENT = "pull_request" + +export const OUTPUT_REASONS = { + "non-pr-event": "Only pull_request events may skip; merge queue, push, and scheduled runs keep full validation.", + "empty-change-range": "The change range contained no files; an empty enumeration is treated as uncertain.", + "unrecognized-status": "The diff contained a change status the classifier does not understand.", + "validation-policy-change": "The classifier or validation gate changed; full validation is required.", + "non-documentation-path": "The change range contained paths outside the documentation-only allowlist.", + "protected-path-segment": + "An allowlisted path contained a protected segment (fixtures, tests, generated, examples).", + "override-label": "Full validation was forced by the ci:full-validation pull request label.", + "override-variable": "Full validation was forced by the ZOO_CODE_REQUIRE_FULL_VALIDATION repository variable.", +} + +export function allowlistFingerprint() { + return createHash("sha256") + .update( + JSON.stringify({ + files: DOCS_ONLY_FILES, + prefixes: DOCS_ONLY_MARKDOWN_PREFIXES, + never: NEVER_SKIP_SEGMENTS, + }), + ) + .digest("hex") + .slice(0, 12) +} + +function hasProtectedSegment(filePath) { + return NEVER_SKIP_SEGMENTS.some((segment) => filePath.split("/").includes(segment)) +} + +export function isDocsOnlyPath(filePath) { + if (typeof filePath !== "string" || filePath.length === 0) return false + const normalized = path.posix.normalize(filePath) + if (normalized.startsWith("../") || path.posix.isAbsolute(normalized)) return false + if (hasProtectedSegment(normalized)) return false + if (DOCS_ONLY_FILES.some((entry) => entry.path === normalized)) return true + if (!normalized.endsWith(".md")) return false + return DOCS_ONLY_MARKDOWN_PREFIXES.some((entry) => normalized.startsWith(entry.prefix)) +} + +// Parses `git diff --name-status -z --find-renames` output. Mirrors the +// parser in scripts/stryker-diff.mjs (rename and copy entries carry the +// old and the new path; all other statuses carry one path) but stays +// self-contained so classification can run without installed +// dependencies. +export function parseNameStatus(output) { + const tokens = output.split("\0") + const entries = [] + + for (let index = 0; index < tokens.length;) { + const statusToken = tokens[index++] + if (!statusToken) break + + const status = statusToken[0] + if (status === "R" || status === "C") { + const oldPath = tokens[index++] + const newPath = tokens[index++] + if (!oldPath || !newPath) throw new Error(`Malformed git name-status entry: ${statusToken}`) + entries.push({ status, oldPath, path: newPath }) + } else { + const filePath = tokens[index++] + if (!filePath) throw new Error(`Malformed git name-status entry: ${statusToken}`) + entries.push({ status, path: filePath }) + } + } + + return entries +} + +function underAllowlistedPrefix(filePath) { + return DOCS_ONLY_MARKDOWN_PREFIXES.some((entry) => filePath.startsWith(entry.prefix)) +} + +// Pure classification over an already-parsed change list. Unknown +// statuses, empty ranges, and any non-allowlisted path fail safe to +// full validation. +export function classifyEntries(entries) { + if (entries.length === 0) { + return { category: "full", reasonCode: "empty-change-range", changedFiles: 0, docsFiles: [], otherFiles: [] } + } + + if (entries.some((entry) => !KNOWN_STATUSES.has(entry.status[0]))) { + return { + category: "full", + reasonCode: "unrecognized-status", + changedFiles: entries.length, + docsFiles: [], + otherFiles: entries.map(({ path: filePath }) => filePath), + } + } + + // Policy changes cannot grant themselves a skip by widening the allowlist. + const policyPaths = entries + .flatMap((entry) => [entry.path, entry.oldPath]) + .filter((filePath) => VALIDATION_POLICY_PATHS.has(filePath)) + if (policyPaths.length > 0) { + return { + category: "full", + reasonCode: "validation-policy-change", + changedFiles: entries.length, + docsFiles: [], + offendingPaths: policyPaths, + } + } + + const docsFiles = [] + const otherFiles = [] + + for (const entry of entries) { + const paths = [entry.path, ...(entry.oldPath ? [entry.oldPath] : [])] + for (const filePath of paths) { + const target = isDocsOnlyPath(filePath) ? docsFiles : otherFiles + target.push({ status: entry.status, path: filePath }) + } + } + + if (otherFiles.length > 0) { + const offendingPaths = otherFiles.map(({ path: filePath }) => filePath) + const allProtected = offendingPaths.every( + (filePath) => hasProtectedSegment(filePath) && underAllowlistedPrefix(filePath), + ) + return { + category: "full", + reasonCode: allProtected ? "protected-path-segment" : "non-documentation-path", + changedFiles: entries.length, + docsFiles, + otherFiles, + offendingPaths: offendingPaths.slice(0, 5), + } + } + + return { category: "docs-only", reasonCode: "allowlisted-documentation", changedFiles: entries.length, docsFiles } +} + +// Classification result for a whole event. Non-pull_request events are +// never skip-eligible; the range is still computed for observability. +export function classifyEvent({ eventName, entries, forceFullReasonCode }) { + const result = classifyEntries(entries) + const skipEligible = eventName === SKIP_ELIGIBLE_EVENT + + if (!skipEligible) { + return { ...result, category: "full", reasonCode: "non-pr-event" } + } + if (forceFullReasonCode && OUTPUT_REASONS[forceFullReasonCode]) { + return { ...result, category: "full", reasonCode: forceFullReasonCode } + } + return result +} + +function git(repoRoot, args) { + return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8", maxBuffer: 20 * 1024 * 1024 }) +} + +export function resolveCommits(repoRoot, baseSha, headSha) { + if (!/^[0-9a-f]{40}$/i.test(baseSha) || !/^[0-9a-f]{40}$/i.test(headSha)) { + throw new Error("Base and head must be full 40-character commit SHAs") + } + for (const [label, sha] of [ + ["base", baseSha], + ["head", headSha], + ]) { + try { + git(repoRoot, ["cat-file", "-e", `${sha}^{commit}`]) + } catch (error) { + throw new Error(`Could not resolve ${label} commit ${sha} in the checkout (shallow or missing history)`) + } + } +} + +export function changedEntries(repoRoot, baseSha, headSha) { + resolveCommits(repoRoot, baseSha, headSha) + let mergeBase + try { + mergeBase = git(repoRoot, ["merge-base", baseSha, headSha]).trim() + git(repoRoot, ["merge-base", "--is-ancestor", mergeBase, headSha]) + } catch (error) { + throw new Error("Could not resolve a common merge base between the base and head commits") + } + const nameStatus = git(repoRoot, ["diff", "--name-status", "-z", "--find-renames", `${mergeBase}...${headSha}`]) + const entries = parseNameStatus(nameStatus) + return { mergeBase, entries } +} + +function writeOutput(name, value) { + if (!process.env.GITHUB_OUTPUT) return + fs.appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`) +} + +function escapeSummaryCell(value) { + return String(value).replaceAll("|", "\\|").replaceAll("\n", " ") +} + +function escapeSummaryPath(value) { + const characters = Array.from(String(value)) + // Encode punctuation before Markdown parsing; cap individual paths in summaries. + const text = characters.slice(0, 500).join("") + return ( + text.replace(/[^\p{L}\p{N} /-]/gu, (character) => `&#${character.codePointAt(0)};`) + + (characters.length > 500 ? "…" : "") + ) +} + +export function formatSummary(result, { eventName, mergeBase, headSha } = {}) { + const lines = [ + "## Change-based validation classification", + "", + "| Field | Value |", + "| --- | --- |", + `| Event | ${escapeSummaryCell(eventName ?? "unknown")} |`, + `| Category | ${escapeSummaryCell(result.category)} |`, + `| Reason | ${escapeSummaryCell(OUTPUT_REASONS[result.reasonCode] ?? result.reasonCode)} |`, + `| Changed files | ${result.changedFiles} |`, + `| Allowlist fingerprint | ${allowlistFingerprint()} |`, + ] + if (headSha) lines.push(`| Head | ${headSha.slice(0, 12)} |`) + if (mergeBase) lines.push(`| Merge base | ${mergeBase.slice(0, 12)} |`) + lines.push("") + + if (result.category === "docs-only") { + lines.push( + "Every changed file matches the documentation-only allowlist. Comprehensive PR validation jobs are skipped; the validation gate verifies this decision.", + "", + "
Allowlisted files", + "", + ) + for (const { path: filePath } of result.docsFiles) lines.push(`- ${escapeSummaryPath(filePath)}`) + lines.push("", "
") + } else if (result.reasonCode === "non-pr-event") { + lines.push( + "Full validation runs unchanged for this event. Merge-queue, release, and push validation is never reduced by classification.", + ) + } else { + lines.push("Full validation runs because the change range is not proven documentation-only.") + const details = result.offendingPaths ?? result.docsFiles.map(({ path: filePath }) => filePath) + if (details.length > 0) { + lines.push("", "
Paths outside or straddling the allowlist", "") + for (const filePath of details) lines.push(`- ${escapeSummaryPath(filePath)}`) + lines.push("", "
") + } + } + return `${lines.join("\n")}\n` +} + +function argument(name) { + const index = process.argv.indexOf(name) + return index === -1 ? undefined : process.argv[index + 1] +} + +function repoRootFromArgv() { + return path.resolve(argument("--repo-root") ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")) +} + +function emitResult(result, context) { + writeOutput("category", result.category) + writeOutput("reason-code", result.reasonCode) + writeOutput("reason", OUTPUT_REASONS[result.reasonCode] ?? result.reasonCode) + writeOutput("changed-files", String(result.changedFiles)) + writeOutput("allowlist-fingerprint", allowlistFingerprint()) + console.log( + `Change classification: ${result.category} (${result.reasonCode}), ${result.changedFiles} changed file(s)`, + ) + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, formatSummary(result, context)) + } +} + +// Walks first-parent history and classifies each commit as if it were a +// pull request change range. Used to validate the allowlist against +// representative historical changes before skips are trusted. +export function runHistory(repoRoot, { limit = 200 } = {}) { + const log = git(repoRoot, ["log", "--first-parent", `--max-count=${limit}`, "--format=%H%x09%s"]) + const commits = log + .split("\n") + .filter(Boolean) + .map((line) => { + const [sha, subject] = line.split("\t") + const prMatch = /\(#(\d+)\)$/.exec(subject) ?? /Merge pull request #(\d+) from/.exec(subject) + return { sha, subject, pr: prMatch ? Number(prMatch[1]) : undefined } + }) + + const rows = [] + for (const commit of commits) { + const parents = git(repoRoot, ["rev-list", "--parents", "-n", "1", commit.sha]).trim().split(/\s+/) + const baseSha = parents[1] + if (!baseSha) continue + try { + const { entries } = changedEntries(repoRoot, baseSha, commit.sha) + const result = classifyEvent({ eventName: SKIP_ELIGIBLE_EVENT, entries }) + rows.push({ ...commit, baseSha, ...result }) + } catch (error) { + rows.push({ + ...commit, + baseSha, + category: "classifier-failure", + reasonCode: error.message, + changedFiles: 0, + }) + } + } + return rows +} + +function formatHistory(rows) { + const skipCount = rows.filter((row) => row.category === "docs-only").length + const lines = [ + `Classified ${rows.length} first-parent commit(s); ${skipCount} would skip comprehensive validation.`, + "", + "| Commit | PR | Category | Reason | Files | Proposed skip file list |", + "| --- | --- | --- | --- | ---: | --- |", + ] + for (const row of rows) { + const skipList = + row.category === "docs-only" + ? (row.docsFiles ?? []) + .map(({ path: filePath }) => escapeSummaryPath(filePath)) + .join(", ") + .slice(0, 180) + : "" + lines.push( + `| ${row.sha.slice(0, 12)} | ${row.pr ?? "—"} | ${row.category} | ${escapeSummaryCell(row.reasonCode)} | ${row.changedFiles} | ${skipList} |`, + ) + } + return `${lines.join("\n")}\n` +} + +function main() { + const repoRoot = repoRootFromArgv() + const command = process.argv[2] + + if (command === "ci") { + const eventName = argument("--event") + const headSha = argument("--head") + if (!eventName || !headSha) + throw new Error( + "Usage: change-classification.mjs ci --event --head [--repo-root ] [--force-full-label] [--force-full-variable]", + ) + const baseSha = argument("--base") ?? git(repoRoot, ["rev-parse", `${headSha}^1`]).trim() + const forceFullReasonCode = process.argv.includes("--force-full-label") + ? "override-label" + : process.argv.includes("--force-full-variable") + ? "override-variable" + : undefined + const { mergeBase, entries } = changedEntries(repoRoot, baseSha, headSha) + const result = classifyEvent({ eventName, entries, forceFullReasonCode }) + emitResult(result, { eventName, mergeBase, headSha }) + return + } + + if (command === "history") { + const limit = Number(argument("--limit") ?? 200) + const rows = runHistory(repoRoot, { limit }) + const output = argument("--output") + if (output) { + fs.mkdirSync(path.dirname(path.resolve(output)), { recursive: true }) + fs.writeFileSync(path.resolve(output), formatHistory(rows)) + const jsonPath = /\.md$/.test(output) + ? path.resolve(output).replace(/\.md$/, ".json") + : `${path.resolve(output)}.json` + fs.writeFileSync(jsonPath, JSON.stringify(rows, null, 2)) + console.log(`History classification written to ${output}`) + } else { + process.stdout.write(formatHistory(rows)) + } + return + } + + throw new Error("Usage: change-classification.mjs [options]") +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main() + } catch (error) { + console.error(`::error title=Change classification::${String(error.message).replaceAll("\n", " ")}`) + console.error(`Change classification failed: ${error.message}`) + process.exitCode = 1 + } +} diff --git a/scripts/change-classification.test.mjs b/scripts/change-classification.test.mjs new file mode 100644 index 0000000000..eab294bba5 --- /dev/null +++ b/scripts/change-classification.test.mjs @@ -0,0 +1,721 @@ +import assert from "node:assert/strict" +import { execFileSync, spawnSync } from "node:child_process" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import { fileURLToPath } from "node:url" + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") +const classifierPath = path.join(repositoryRoot, "scripts/change-classification.mjs") + +import { replayClassification, validationPolicyChanged } from "./change-gate.mjs" + +import { + NEVER_SKIP_SEGMENTS, + resolveCommits, + allowlistFingerprint, + changedEntries, + classifyEntries, + classifyEvent, + formatSummary, + isDocsOnlyPath, + parseNameStatus, + runHistory, +} from "./change-classification.mjs" + +describe("documentation-only allowlist", () => { + it("accepts the enumerated root documentation files", () => { + for (const filePath of ["CONTRIBUTING.md", "SECURITY.md", "PRIVACY.md", "CODE_OF_CONDUCT.md"]) { + assert.ok(isDocsOnlyPath(filePath), `${filePath} should be allowlisted`) + } + }) + + it("accepts markdown under the documentation and template prefixes", () => { + for (const filePath of [ + "docs/development-quality-checks.md", + "docs/architecture/task-lifecycle-model.md", + ".github/pull_request_template.md", + ".github/ISSUE_TEMPLATE/bug_report.md", + ]) { + assert.ok(isDocsOnlyPath(filePath), `${filePath} should be allowlisted`) + } + }) + + it("rejects files that are build, runtime, test, or release inputs", () => { + for (const filePath of [ + "README.md", + "CHANGELOG.md", + "src/CHANGELOG.md", + "AGENTS.md", + "src/README.md", + "locales/en/README.md", + "package.json", + "pnpm-lock.yaml", + "docs/image.png", + "docs/scripts/example.js", + ".github/workflows/code-qa.yml", + ".github/CODEOWNERS", + ".github/actions/classify-changes/action.yml", + "src/core/task/index.ts", + ]) { + assert.equal(isDocsOnlyPath(filePath), false, `${filePath} must not be allowlisted`) + } + }) + + it("never skips protected path segments even under allowlisted prefixes", () => { + for (const filePath of [ + "docs/fixtures/sample.md", + "docs/__tests__/case.md", + "docs/generated/report.md", + "docs/examples/walkthrough.md", + "docs/test-utils/guide.md", + ".github/ISSUE_TEMPLATE/fixtures/template.md", + ]) { + assert.equal(isDocsOnlyPath(filePath), false, `${filePath} must not be allowlisted`) + } + }) + + it("rejects traversal, absolute paths, and empty values", () => { + for (const filePath of ["../outside.md", "/etc/passwd.md", "", undefined, null]) { + assert.equal(isDocsOnlyPath(filePath), false) + } + }) +}) + +for (const segment of NEVER_SKIP_SEGMENTS) { + it(`protects segment ${segment}`, () => { + for (const prefix of ["docs", ".github"]) assert.equal(isDocsOnlyPath(`${prefix}/${segment}/case.md`), false) + }) +} + +describe("name-status parsing", () => { + it("parses single-path entries", () => { + assert.deepEqual(parseNameStatus("A\0docs/new.md\0M\0CONTRIBUTING.md\0D\0src/gone.ts\0"), [ + { status: "A", path: "docs/new.md" }, + { status: "M", path: "CONTRIBUTING.md" }, + { status: "D", path: "src/gone.ts" }, + ]) + }) + + it("parses rename entries with old and new paths", () => { + assert.deepEqual(parseNameStatus("R100\0docs/old.md\0docs/new.md\0M\0src/a.ts\0"), [ + { status: "R", oldPath: "docs/old.md", path: "docs/new.md" }, + { status: "M", path: "src/a.ts" }, + ]) + }) + + it("rejects malformed entries", () => { + assert.throws(() => parseNameStatus("R100\0only-one-path\0")) + assert.throws(() => parseNameStatus("M\0")) + }) +}) + +describe("change classification", () => { + it("fails safe on an empty change range", () => { + const result = classifyEntries([]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "empty-change-range") + }) + + it("classifies an all-documentation range including deletions and renames", () => { + const result = classifyEntries([ + { status: "A", path: "docs/new.md" }, + { status: "M", path: "CONTRIBUTING.md" }, + { status: "D", path: "docs/gone.md" }, + { status: "R", oldPath: "docs/old.md", path: "docs/renamed.md" }, + { status: "T", path: "docs/symlinked.md" }, + ]) + assert.equal(result.category, "docs-only") + assert.equal(result.changedFiles, 5) + }) + + it("fails safe when a rename moves documentation into source", () => { + const result = classifyEntries([{ status: "R", oldPath: "docs/old.md", path: "src/moved.md" }]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "non-documentation-path") + assert.ok(result.offendingPaths.includes("src/moved.md")) + }) + + it("fails safe when a rename moves source into documentation", () => { + const result = classifyEntries([{ status: "R", oldPath: "src/moved.ts", path: "docs/moved.md" }]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "non-documentation-path") + assert.ok(result.offendingPaths.includes("src/moved.ts")) + }) + + it("fails safe on mixed documentation and source changes", () => { + const result = classifyEntries([ + { status: "M", path: "CONTRIBUTING.md" }, + { status: "M", path: "src/core/task/index.ts" }, + ]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "non-documentation-path") + }) + + it("fails safe on protected segments inside the documentation tree", () => { + const result = classifyEntries([{ status: "R", oldPath: "docs/a.md", path: "docs/fixtures/b.md" }]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "protected-path-segment") + }) + + it("fails safe on unrecognized change statuses", () => { + const result = classifyEntries([{ status: "C", oldPath: "docs/a.md", path: "docs/b.md" }]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "unrecognized-status") + }) +}) + +describe("event gating", () => { + const docsEntries = [{ status: "M", path: "docs/only.md" }] + + it("allows a skip only for documentation-only pull request ranges", () => { + assert.equal(classifyEvent({ eventName: "pull_request", entries: docsEntries }).category, "docs-only") + }) + + it("forces full validation for merge queue, push, and scheduled events", () => { + for (const eventName of ["merge_group", "push", "schedule", "workflow_dispatch"]) { + const result = classifyEvent({ eventName, entries: docsEntries }) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "non-pr-event") + } + }) + + it("honors explicit full-validation overrides", () => { + const label = classifyEvent({ + eventName: "pull_request", + entries: docsEntries, + forceFullReasonCode: "override-label", + }) + assert.equal(label.category, "full") + assert.equal(label.reasonCode, "override-label") + + const variable = classifyEvent({ + eventName: "pull_request", + entries: docsEntries, + forceFullReasonCode: "override-variable", + }) + assert.equal(variable.reasonCode, "override-variable") + }) + + it("ignores unknown override codes instead of failing open", () => { + const result = classifyEvent({ eventName: "pull_request", entries: docsEntries, forceFullReasonCode: "bogus" }) + assert.equal(result.category, "docs-only") + }) +}) + +describe("summary rendering", () => { + it("records the category, reason, and allowlist fingerprint", () => { + const summary = formatSummary(classifyEntries([{ status: "M", path: "docs/only.md" }]), { + eventName: "pull_request", + }) + assert.match(summary, /docs-only/) + assert.match(summary, /Change-based validation classification/) + assert.ok(summary.includes(allowlistFingerprint())) + }) +}) + +function git(tempRepo, args) { + return execFileSync("git", ["-c", "user.email=ci@example.com", "-c", "user.name=CI Test", ...args], { + cwd: tempRepo, + encoding: "utf8", + }) +} + +function createTempRepository() { + const tempRepo = fs.mkdtempSync(path.join(os.tmpdir(), "change-classification-")) + git(tempRepo, ["init", "--initial-branch=main"]) + fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 1\n") + fs.mkdirSync(path.join(tempRepo, "docs"), { recursive: true }) + fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "# base\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "base"]) + return tempRepo +} + +describe("history validation", () => { + it("classifies representative commits without errors", () => { + const tempRepo = createTempRepository() + try { + fs.writeFileSync(path.join(tempRepo, "docs/guide.md"), "# guide\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs: add guide (#101)"]) + + fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 2\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "feat: change code (#102)"]) + + fs.writeFileSync(path.join(tempRepo, "docs/mixed.md"), "# mixed\n") + fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 3\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "mixed: docs and code (#103)"]) + + const rows = runHistory(tempRepo, { limit: 10 }) + const bySubject = new Map(rows.map((row) => [row.subject, row])) + assert.equal(bySubject.get("docs: add guide (#101)").category, "docs-only") + assert.equal(bySubject.get("feat: change code (#102)").category, "full") + assert.equal(bySubject.get("mixed: docs and code (#103)").category, "full") + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) +}) + +describe("ci command", () => { + function runCi(tempRepo, args) { + const githubOutput = path.join(tempRepo, "github-output.env") + const stepSummary = path.join(tempRepo, "step-summary.md") + const result = spawnSync(process.execPath, [classifierPath, "ci", ...args], { + cwd: repositoryRoot, + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: githubOutput, + GITHUB_STEP_SUMMARY: stepSummary, + }, + }) + return { result, githubOutput, stepSummary } + } + + function readOutputEnv(filePath) { + const entries = new Map() + for (const line of fs.readFileSync(filePath, "utf8").split("\n").filter(Boolean)) { + const index = line.indexOf("=") + entries.set(line.slice(0, index), line.slice(index + 1)) + } + return entries + } + + it("classifies a documentation-only merge commit and emits workflow outputs", () => { + const tempRepo = createTempRepository() + try { + git(tempRepo, ["checkout", "-b", "docs-change"]) + fs.writeFileSync(path.join(tempRepo, "docs/pr.md"), "# pr\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs change"]) + git(tempRepo, ["checkout", "main"]) + git(tempRepo, ["merge", "--no-ff", "-m", "merge docs change", "docs-change"]) + const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + + const { result, githubOutput, stepSummary } = runCi(tempRepo, [ + "--event", + "pull_request", + "--head", + headSha, + "--repo-root", + tempRepo, + ]) + assert.equal(result.status, 0, result.stderr) + const outputs = readOutputEnv(githubOutput) + assert.equal(outputs.get("category"), "docs-only") + assert.equal(outputs.get("changed-files"), "1") + const summary = fs.readFileSync(stepSummary, "utf8") + assert.match(summary, /docs-only/) + assert.ok(summary.includes("docs/pr.md")) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) + + for (const [flags, reason] of [ + [["--force-full-label"], "override-label"], + [["--force-full-variable"], "override-variable"], + [["--force-full-label", "--force-full-variable"], "override-label"], + ]) { + it(`honors CLI overrides ${flags.join(" ")}`, () => { + const tempRepo = createTempRepository() + try { + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + const { result, githubOutput } = runCi(tempRepo, [ + "--event", + "pull_request", + "--base", + base, + "--head", + head, + "--repo-root", + tempRepo, + ...flags, + ]) + assert.equal(result.status, 0, result.stderr) + assert.equal(readOutputEnv(githubOutput).get("category"), "full") + assert.equal(readOutputEnv(githubOutput).get("reason-code"), reason) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) + } + it("forces full validation for non-pull_request events", () => { + const tempRepo = createTempRepository() + try { + git(tempRepo, ["checkout", "-b", "docs-change"]) + fs.writeFileSync(path.join(tempRepo, "docs/pr.md"), "# pr\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs change"]) + git(tempRepo, ["checkout", "main"]) + git(tempRepo, ["merge", "--no-ff", "-m", "merge docs change", "docs-change"]) + const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + + const { result, githubOutput } = runCi(tempRepo, [ + "--event", + "merge_group", + "--head", + headSha, + "--repo-root", + tempRepo, + ]) + assert.equal(result.status, 0, result.stderr) + assert.equal(readOutputEnv(githubOutput).get("reason-code"), "non-pr-event") + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) + + it("fails closed when the head commit is missing from history", () => { + const tempRepo = createTempRepository() + try { + const { result } = runCi(tempRepo, [ + "--event", + "pull_request", + "--head", + "0".repeat(40), + "--repo-root", + tempRepo, + ]) + assert.equal(result.status, 1) + assert.match(result.stderr, /::error/) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) +}) + +describe("history report files", () => { + for (const filename of ["report.md", "report.txt", "report", "report.json"]) { + it(`preserves the Markdown report and JSON sidecar for ${filename}`, () => { + const tempRepo = createTempRepository() + try { + const output = path.join(tempRepo, filename) + const result = spawnSync( + process.execPath, + [classifierPath, "history", "--repo-root", tempRepo, "--output", output], + { encoding: "utf8" }, + ) + assert.equal(result.status, 0, result.stderr) + assert.match(fs.readFileSync(output, "utf8"), /Classified .*first-parent commit/) + const jsonPath = filename.endsWith(".md") ? output.replace(/\.md$/, ".json") : `${output}.json` + assert.deepEqual(JSON.parse(fs.readFileSync(jsonPath, "utf8")), []) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) + } +}) + +describe("validation policy changes", () => { + for (const filePath of ["scripts/change-classification.mjs", "scripts/change-gate.mjs"]) { + for (const entry of [ + { status: "M", path: filePath }, + { status: "R", oldPath: filePath, path: "docs/policy.md" }, + ]) { + it(`requires full validation for ${entry.status} ${filePath}`, () => { + const result = classifyEntries([entry]) + assert.equal(result.category, "full") + assert.equal(result.reasonCode, "validation-policy-change") + }) + } + } + + it("independently detects policy changes even when the classifier reports docs-only", () => { + const tempRepo = createTempRepository() + try { + const baseSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + assert.equal(validationPolicyChanged(tempRepo, baseSha, baseSha), false) + fs.mkdirSync(path.join(tempRepo, "scripts")) + fs.writeFileSync( + path.join(tempRepo, "scripts/change-classification.mjs"), + 'console.log("Change classification: docs-only (allowlisted-documentation), 1 changed file(s)")', + ) + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "widen classification"]) + const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + assert.equal( + replayClassification(tempRepo, { baseSha, headSha, eventName: "pull_request" }).classification.category, + "docs-only", + ) + assert.equal(validationPolicyChanged(tempRepo, baseSha, headSha), true) + assert.equal(validationPolicyChanged(tempRepo, "0".repeat(40), headSha), true) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) + + it("replays the complete explicit base range rather than just the last commit", () => { + const tempRepo = createTempRepository() + try { + fs.mkdirSync(path.join(tempRepo, "scripts")) + fs.copyFileSync(classifierPath, path.join(tempRepo, "scripts/change-classification.mjs")) + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "install classifier"]) + const baseSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "export const base = 2\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "code change"]) + fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "# updated\n") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs change"]) + const headSha = git(tempRepo, ["rev-parse", "HEAD"]).trim() + const replay = replayClassification(tempRepo, { baseSha, headSha, eventName: "pull_request" }) + assert.equal(replay.status, 0) + assert.equal(replay.classification.category, "full") + assert.equal(replay.classification.changedFiles, 2) + assert.equal(validationPolicyChanged(tempRepo, baseSha, headSha), false) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) +}) + +it("escapes filenames in classification summaries", () => { + const filename = "docs/`
|\nforged.md" + const summary = formatSummary(classifyEntries([{ status: "M", path: filename }])) + assert.ok(summary.includes("docs/`<details>| forged.md")) + assert.ok(!summary.includes(filename)) +}) + +it("pins the complete allowlist fingerprint including protected segments", () => { + assert.equal(allowlistFingerprint(), "15dc52b392ab") +}) + +for (const sha of ["", "abc", "g".repeat(40), "a".repeat(39), "a".repeat(41)]) { + it(`rejects invalid SHA ${sha}`, () => { + assert.throws(() => resolveCommits(repositoryRoot, sha, "a".repeat(40)), /40-character/) + assert.throws(() => resolveCommits(repositoryRoot, "a".repeat(40), sha), /40-character/) + assert.equal(validationPolicyChanged(repositoryRoot, sha, "a".repeat(40)), true) + assert.equal(validationPolicyChanged(repositoryRoot, "a".repeat(40), sha), true) + }) +} + +it("runs the gate CLI against a docs-only repository and writes its summary", () => { + const tempRepo = createTempRepository() + try { + fs.mkdirSync(path.join(tempRepo, "scripts")) + for (const file of ["change-classification.mjs", "change-gate.mjs"]) + fs.copyFileSync(path.join(repositoryRoot, "scripts", file), path.join(tempRepo, "scripts", file)) + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "install validation scripts"]) + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + const cases = [ + { name: "docs skip", actual: "skipped", status: 0 }, + { name: "unexpected run", actual: "success", status: 1 }, + { name: "label", actual: "success", category: "full", reason: "override-label", label: "true", status: 0 }, + { + name: "variable", + actual: "success", + category: "full", + reason: "override-variable", + variable: "true", + status: 0, + }, + { + name: "label wins", + actual: "success", + category: "full", + reason: "override-label", + label: "true", + variable: "true", + status: 0, + }, + { + name: "draft", + actual: "skipped", + rule: "pr-non-draft", + draft: "true", + category: "full", + reason: "override-label", + label: "true", + status: 0, + }, + { + name: "ready", + actual: "success", + rule: "pr-non-draft", + category: "full", + reason: "override-label", + label: "true", + status: 0, + }, + { name: "missing event", actual: "skipped", event: "", error: /EVENT_NAME is required/, status: 1 }, + { name: "missing watches", actual: "skipped", watch: "", error: /WATCHED_JOBS is empty/, status: 1 }, + { name: "missing needs", actual: "skipped", needs: "{}", error: /missing/, status: 1 }, + { name: "missing base", actual: "skipped", base: "", error: /--base and --head are required/, status: 1 }, + ] + for (const testCase of cases) { + const { actual, status } = testCase + const summary = path.join(tempRepo, `summary-${testCase.name}.md`) + const result = spawnSync( + process.execPath, + [ + path.join(tempRepo, "scripts/change-gate.mjs"), + "ci", + "--base", + testCase.base ?? base, + "--head", + head, + "--skip-metrics", + ], + { + encoding: "utf8", + env: { + ...process.env, + EVENT_NAME: testCase.event ?? "pull_request", + WATCHED_JOBS: testCase.watch ?? `expensive:${testCase.rule ?? "always"}:gated`, + NEEDS_JSON: + testCase.needs ?? + JSON.stringify({ + classify: { + result: "success", + outputs: { + category: testCase.category ?? "docs-only", + "reason-code": testCase.reason ?? "allowlisted-documentation", + }, + }, + expensive: { result: actual }, + }), + GITHUB_STEP_SUMMARY: summary, + FORCE_FULL_LABEL: testCase.label ?? "false", + PR_DRAFT: testCase.draft ?? "false", + FORCE_FULL_VARIABLE: testCase.variable ?? "false", + }, + }, + ) + assert.equal(result.status, status, `${testCase.name}: ${result.stderr}`) + if (testCase.error) { + assert.match(result.stderr, testCase.error) + continue + } + assert.match(fs.readFileSync(summary, "utf8"), /Validation gate/) + assert.match(fs.readFileSync(summary, "utf8"), /Runner-minute metrics were unavailable/) + } + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } +}) + +for (const source of ["process.exit(1)", 'console.log("malformed")']) { + it(`rejects a failed or malformed classifier replay: ${source}`, () => { + const tempRepo = createTempRepository() + try { + fs.mkdirSync(path.join(tempRepo, "scripts")) + fs.writeFileSync(path.join(tempRepo, "scripts/change-classification.mjs"), source) + assert.deepEqual( + replayClassification(tempRepo, { + baseSha: "a".repeat(40), + headSha: "b".repeat(40), + eventName: "pull_request", + }), + { status: 1, classification: undefined }, + ) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) +} + +it("uses the PR merge base and detects documentation renames", () => { + const tempRepo = createTempRepository() + try { + git(tempRepo, ["checkout", "-b", "pr"]) + git(tempRepo, ["mv", "docs/base.md", "docs/renamed.md"]) + git(tempRepo, ["commit", "-m", "rename docs"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + git(tempRepo, ["checkout", "main"]) + fs.writeFileSync(path.join(tempRepo, "src-code.ts"), "main-only change") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "advance main"]) + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + assert.deepEqual(changedEntries(tempRepo, base, head).entries, [ + { status: "R", oldPath: "docs/base.md", path: "docs/renamed.md" }, + ]) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } +}) + +for (const filename of ["docs/space name.md", "docs/日本語.md", "docs/line\nbreak.md", "docs/UPPER.MD"]) { + it(`classifies the real git path ${JSON.stringify(filename)}`, () => { + const tempRepo = createTempRepository() + try { + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.writeFileSync(path.join(tempRepo, filename), "# documentation") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "add unusual path"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + const { entries } = changedEntries(tempRepo, base, head) + assert.deepEqual(entries, [{ status: "A", path: filename }]) + assert.equal(classifyEntries(entries).category, filename.endsWith(".MD") ? "full" : "docs-only") + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } + }) +} + +it("requires full validation for a real documentation-to-source rename", () => { + const tempRepo = createTempRepository() + try { + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.mkdirSync(path.join(tempRepo, "src")) + git(tempRepo, ["mv", "docs/base.md", "src/source.md"]) + git(tempRepo, ["commit", "-m", "move documentation into source"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + const { entries } = changedEntries(tempRepo, base, head) + assert.deepEqual(entries, [{ status: "R", oldPath: "docs/base.md", path: "src/source.md" }]) + assert.equal(classifyEntries(entries).category, "full") + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + } +}) + +it("fails closed when a shallow clone lacks the base commit", () => { + const tempRepo = createTempRepository() + const clone = fs.mkdtempSync(path.join(os.tmpdir(), "classification-shallow-")) + try { + const base = git(tempRepo, ["rev-parse", "HEAD"]).trim() + fs.writeFileSync(path.join(tempRepo, "docs/base.md"), "updated") + git(tempRepo, ["add", "."]) + git(tempRepo, ["commit", "-m", "docs"]) + const head = git(tempRepo, ["rev-parse", "HEAD"]).trim() + git(tempRepo, ["clone", "--depth=1", `file://${tempRepo}`, clone]) + assert.equal(git(clone, ["rev-parse", "--is-shallow-repository"]).trim(), "true") + assert.throws(() => changedEntries(clone, base, head), /Could not resolve base commit/) + assert.equal(validationPolicyChanged(clone, base, head), true) + } finally { + fs.rmSync(tempRepo, { recursive: true, force: true }) + fs.rmSync(clone, { recursive: true, force: true }) + } +}) + +it("renders formatting and autolink punctuation as literal filename characters", () => { + const filename = "docs/[link](https://example.com)*_~#@!\\.md" + const summary = formatSummary(classifyEntries([{ status: "M", path: filename }])) + const line = summary.split("\n").find((line) => line.startsWith("- docs/")) + assert.equal( + line, + "- docs/[link](https://example.com)*_~#@!\.md", + ) +}) + +it("caps long filename summaries", () => { + const filename = `docs/${"a".repeat(600)}.md` + const summary = formatSummary(classifyEntries([{ status: "M", path: filename }])) + const line = summary.split("\n").find((line) => line.startsWith("- docs/")) + assert.equal(line, `- docs/${"a".repeat(495)}…`) +}) diff --git a/scripts/change-gate.mjs b/scripts/change-gate.mjs new file mode 100644 index 0000000000..124013b11f --- /dev/null +++ b/scripts/change-gate.mjs @@ -0,0 +1,376 @@ +#!/usr/bin/env node + +// Always-running validation gate for change-based CI (issue #2291). +// +// The gate runs after the classification and every validation job of its +// workflow and rejects incoherent outcomes: +// - a missing, malformed, or non-terminal classification, +// - validation jobs that did not run when full validation was required, +// - validation jobs that ran when a documentation-only skip was expected, +// - a classifier replay that disagrees with the recorded classification. +// +// It also records the observability contract: classified category, the +// skipped work with explanations, runner minutes per job, and the time +// until the gate completed. + +import { spawnSync } from "node:child_process" +import fs from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" + +const WATCH_RULES = new Set(["always", "pr-only", "pr-non-draft"]) +const TERMINAL_RESULTS = new Set(["success", "failure", "skipped", "cancelled"]) + +export const SKIP_EXPLANATION = + "Documentation-only change: every changed file matches the allowlist, so this job skipped expensive validation." +export const FALLBACK_EXPLANATION = "Classification was unavailable, so the fail-safe default selected full validation." + +// `name:rule:gated` triples describing the jobs this gate watches. +export function parseWatchSpec(spec) { + if (typeof spec !== "string" || spec.trim().length === 0) throw new Error("WATCHED_JOBS is empty") + return spec.split(",").map((entry) => { + const parts = entry.trim().split(":") + if (parts.length !== 3) throw new Error(`Malformed watch entry: ${entry}`) + const [name, rule, gated] = parts + if (!WATCH_RULES.has(rule)) throw new Error(`Unknown watch rule in entry: ${entry}`) + if (gated !== "gated" && gated !== "ungated") throw new Error(`Unknown gating flag in entry: ${entry}`) + return { name, rule, gated: gated === "gated" } + }) +} + +export function isDocsOnlySkip(needsEntry, eventName) { + return ( + needsEntry?.result === "success" && + needsEntry?.outputs?.category === "docs-only" && + eventName === "pull_request" + ) +} + +export function expectedResult({ rule, gated }, { eventName, prDraft, docsOnly }) { + if (rule === "pr-only" && eventName !== "pull_request") return "skipped" + if (rule === "pr-non-draft" && eventName === "pull_request" && prDraft) return "skipped" + if (docsOnly && gated && eventName === "pull_request") return "skipped" + return "success" +} + +function explanationFor(expected, { docsOnly, classificationFailed }) { + if (expected === "skipped") return docsOnly ? SKIP_EXPLANATION : "Job has its own event or draft condition." + if (classificationFailed) return FALLBACK_EXPLANATION + return "Full validation ran." +} + +// Compares the needs graph with the outcome implied by the +// classification. Missing results, failed full validation, and jobs +// whose skip state contradicts the classification are all violations. +export function evaluateGate({ needs, watched, eventName, prDraft, classifierReplay, policyChanged = false }) { + const violations = [] + const notes = [] + + const classify = needs?.classify + if (!classify || typeof classify.result !== "string") { + violations.push("Classification job result is missing from the needs graph.") + } else if (!TERMINAL_RESULTS.has(classify.result)) { + violations.push(`Classification job reported non-terminal result: ${classify.result}`) + } else if (classify.result !== "success" && classify.result !== "failure") { + violations.push(`Classification must either succeed or fail; got: ${classify.result}`) + } + + const classificationFailed = classify?.result === "failure" + if (classify?.result === "success") { + const category = classify.outputs?.category + if (category !== "docs-only" && category !== "full") { + violations.push(`Classification output category is malformed: ${category}`) + } + } + + const docsOnly = isDocsOnlySkip(classify, eventName) + if (docsOnly && policyChanged) { + violations.push( + "Validation policy changed or could not be verified against the base; full validation is required.", + ) + } + + if (classifierReplay) { + if (classifierReplay.status === 0) { + const replay = classifierReplay.classification + if (classify?.result === "failure") { + violations.push( + "The gate reproduced a classification the classify job reported as failed; the runs are inconsistent.", + ) + } else if ( + replay?.category !== classify?.outputs?.category || + replay?.reasonCode !== classify?.outputs?.["reason-code"] + ) { + violations.push( + `Classifier replay disagrees with the recorded classification: replay=${replay?.category}/${replay?.reasonCode} recorded=${classify?.outputs?.category}/${classify?.outputs?.["reason-code"]}`, + ) + } + } else if (classify?.result === "success") { + violations.push( + "The gate could not reproduce the classification that the classify job reported; refusing to trust the recorded category.", + ) + } else { + notes.push( + "Both the classify job and the gate replay failed; full validation ran by the fail-safe default.", + ) + } + } + + const rows = [] + for (const job of watched) { + const actual = needs?.[job.name]?.result + const expected = expectedResult(job, { eventName, prDraft, docsOnly }) + const ok = actual === expected + if (!actual) { + violations.push(`Watched job ${job.name} has no result in the needs graph (missing or misnamed).`) + } else if (!ok) { + violations.push( + expected === "success" + ? `Required validation missing: ${job.name} reported "${actual}" but full validation was required.` + : `Unexpected run: ${job.name} reported "${actual}" but the classification expected it to be skipped.`, + ) + } + rows.push({ + ...job, + expected, + actual: actual ?? "missing", + ok, + explanation: explanationFor(expected, { docsOnly, classificationFailed }), + }) + } + + return { ok: violations.length === 0, violations, notes, rows, docsOnly, classificationFailed } +} + +// Best-effort runner-minute metrics from the Actions API. `fetchJson` +// is injectable for tests. +export function collectJobMetrics({ repository, runId, fetchJson }) { + const run = fetchJson(`repos/${repository}/actions/runs/${runId}`) + const jobs = [] + let page = 1 + for (;;) { + const response = fetchJson(`repos/${repository}/actions/runs/${runId}/jobs?per_page=100&page=${page}`) + jobs.push(...(response.jobs ?? [])) + if ((response.jobs ?? []).length < 100) break + page += 1 + } + + const minutes = (start, end) => Math.max(0, (new Date(end) - new Date(start)) / 60_000) + const perJob = jobs + .filter((job) => job.started_at && job.completed_at) + .map((job) => ({ name: job.name, runnerMinutes: Number(minutes(job.started_at, job.completed_at).toFixed(2)) })) + const totalRunnerMinutes = Number(perJob.reduce((sum, job) => sum + job.runnerMinutes, 0).toFixed(2)) + const latestCompletion = jobs + .map((job) => job.completed_at) + .filter(Boolean) + .sort() + .at(-1) + const completionMinutes = + run?.started_at && latestCompletion ? Number(minutes(run.started_at, latestCompletion).toFixed(2)) : undefined + + return { perJob, totalRunnerMinutes, completionMinutes } +} + +function escapeSummaryCell(value) { + return String(value).replaceAll("|", "\\|").replaceAll("\n", " ") +} + +export function renderGateSummary({ rows, needs, eventName, metrics, violations, notes }) { + const classify = needs?.classify + const lines = [ + "## Validation gate", + "", + "| Field | Value |", + "| --- | --- |", + `| Event | ${escapeSummaryCell(eventName)} |`, + `| Classification | ${escapeSummaryCell(classify?.outputs?.category ?? "unavailable")} (${escapeSummaryCell(classify?.outputs?.["reason-code"] ?? "no reason recorded")}) |`, + `| Classification job | ${escapeSummaryCell(classify?.result ?? "missing")} |`, + `| Skip decision | ${rows.some((row) => row.expected === "skipped" && row.explanation === SKIP_EXPLANATION) ? "documentation-only jobs skipped" : "no classification-based skips"} |`, + `| Gate outcome | ${violations.length === 0 ? "passed" : "failed"} |`, + "", + "| Job | Watch rule | Gated | Expected | Actual | Explanation |", + "| --- | --- | --- | --- | --- | --- |", + ] + for (const row of rows) { + lines.push( + `| ${escapeSummaryCell(row.name)} | ${row.rule} | ${row.gated ? "yes" : "no"} | ${row.expected} | ${escapeSummaryCell(row.actual)} | ${escapeSummaryCell(row.explanation)} |`, + ) + } + + if (metrics) { + lines.push( + "", + "| Runner metric | Minutes |", + "| --- | ---: |", + `| Total runner minutes (this workflow run) | ${metrics.totalRunnerMinutes} |`, + ...(metrics.completionMinutes === undefined + ? [] + : [`| Required-check completion time (run start to last job) | ${metrics.completionMinutes} |`]), + "", + "
Runner minutes per job", + "", + "| Job | Minutes |", + "| --- | ---: |", + ...metrics.perJob.map((job) => `| ${escapeSummaryCell(job.name)} | ${job.runnerMinutes} |`), + "", + "
", + ) + } else { + lines.push("", "Runner-minute metrics were unavailable for this run (best effort only).") + } + + if (notes.length > 0) lines.push("", ...notes.map((note) => `Note: ${note}`)) + if (violations.length > 0) { + lines.push("", "### Gate violations", "") + for (const violation of violations) lines.push(`- ${violation}`) + } + return `${lines.join("\n")}\n` +} + +function repositoryRoot() { + return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") +} + +export function parseClassifierStdout(stdout) { + const match = /Change classification: (\S+) \(([^)]+)\), (\d+) changed file/.exec(stdout ?? "") + return match ? { category: match[1], reasonCode: match[2], changedFiles: Number(match[3]) } : undefined +} + +// Check git directly, independently of the head classifier's allowlist or replay. +// Missing history and git errors fail safe by refusing documentation-only skips. +export function validationPolicyChanged(repoRoot, baseSha, headSha) { + if (![baseSha, headSha].every((sha) => /^[0-9a-f]{40}$/i.test(sha ?? ""))) return true + const result = spawnSync( + "git", + [ + "diff", + "--quiet", + `${baseSha}...${headSha}`, + "--", + "scripts/change-classification.mjs", + "scripts/change-gate.mjs", + ], + { cwd: repoRoot }, + ) + return Boolean(result.error) || result.status !== 0 +} + +export function replayClassification(repoRoot, { baseSha, headSha, eventName, forceFullLabel, forceFullVariable }) { + const args = [ + path.join(repoRoot, "scripts/change-classification.mjs"), + "ci", + "--event", + eventName, + "--base", + baseSha, + "--head", + headSha, + "--repo-root", + repoRoot, + ] + if (forceFullLabel === "true") args.push("--force-full-label") + if (forceFullVariable === "true") args.push("--force-full-variable") + + const result = spawnSync(process.execPath, args, { + cwd: repoRoot, + encoding: "utf8", + // The replay must not write workflow outputs or summaries. + env: { ...process.env, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" }, + }) + if (result.status !== 0) return { status: result.status ?? 1, classification: undefined } + + const classification = parseClassifierStdout(result.stdout) + return classification ? { status: 0, classification } : { status: 1, classification: undefined } +} + +function argument(name) { + const index = process.argv.indexOf(name) + return index === -1 ? undefined : process.argv[index + 1] +} + +function main() { + const repoRoot = repositoryRoot() + const command = process.argv[2] + if (command !== "ci") throw new Error("Usage: change-gate.mjs ci --base --head [--skip-metrics]") + + const headSha = argument("--head") + const baseSha = argument("--base") + if (!headSha || !baseSha) throw new Error("--base and --head are required") + + const eventName = process.env.EVENT_NAME + if (!eventName) throw new Error("EVENT_NAME is required") + const watched = parseWatchSpec(process.env.WATCHED_JOBS ?? "") + const needs = JSON.parse(process.env.NEEDS_JSON ?? "{}") + + const classifierReplay = replayClassification(repoRoot, { + baseSha, + headSha, + eventName, + forceFullLabel: process.env.FORCE_FULL_LABEL, + forceFullVariable: process.env.FORCE_FULL_VARIABLE, + }) + if (classifierReplay.classification === undefined && classifierReplay.status === 0) { + classifierReplay.status = 1 + } + + const evaluation = evaluateGate({ + needs, + watched, + eventName, + prDraft: process.env.PR_DRAFT === "true", + policyChanged: validationPolicyChanged(repoRoot, baseSha, headSha), + classifierReplay: { status: classifierReplay.status, classification: classifierReplay.classification }, + }) + + let metrics + if (!process.argv.includes("--skip-metrics") && process.env.GH_TOKEN && process.env.RUN_ID) { + try { + metrics = collectJobMetrics({ + repository: process.env.GITHUB_REPOSITORY, + runId: process.env.RUN_ID, + fetchJson: (route) => { + const result = spawnSync("gh", ["api", route], { + encoding: "utf8", + env: process.env, + maxBuffer: 20 * 1024 * 1024, + }) + if (result.status !== 0) throw new Error(result.stderr) + return JSON.parse(result.stdout) + }, + }) + } catch (error) { + console.warn( + `::warning title=Validation gate::Runner-minute metrics unavailable: ${String(error.message).replaceAll("\n", " ")}`, + ) + } + } + + if (process.env.GITHUB_STEP_SUMMARY) { + fs.appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + renderGateSummary({ ...evaluation, needs, eventName, metrics }), + ) + } + + for (const note of evaluation.notes) console.log(`Validation gate note: ${note}`) + for (const violation of evaluation.violations) { + console.error(`::error title=Validation gate::${violation.replaceAll("\n", " ")}`) + } + if (evaluation.ok) { + console.log( + `Validation gate passed: classification=${needs?.classify?.outputs?.category ?? "unavailable"}, watched=${evaluation.rows.length} job(s).`, + ) + return + } + process.exitCode = 1 +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + main() + } catch (error) { + console.error(`::error title=Validation gate::${String(error.message).replaceAll("\n", " ")}`) + console.error(`Validation gate failed: ${error.message}`) + process.exitCode = 1 + } +} diff --git a/scripts/change-gate.test.mjs b/scripts/change-gate.test.mjs new file mode 100644 index 0000000000..fa5448975a --- /dev/null +++ b/scripts/change-gate.test.mjs @@ -0,0 +1,544 @@ +import assert from "node:assert/strict" +import { parse } from "yaml" +import fs from "node:fs" +import path from "node:path" +import { describe, it } from "node:test" +import { fileURLToPath } from "node:url" + +const repositoryRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") + +import { + collectJobMetrics, + evaluateGate, + expectedResult, + parseWatchSpec, + parseClassifierStdout, + renderGateSummary, +} from "./change-gate.mjs" + +const needsFor = (results, classify) => ({ + classify: classify ?? { result: "success", outputs: { category: "full", "reason-code": "non-documentation-path" } }, + ...Object.fromEntries(Object.entries(results).map(([name, result]) => [name, { result }])), +}) + +describe("watch specification", () => { + it("parses name:rule:gated triples", () => { + assert.deepEqual(parseWatchSpec("job-a:always:gated, job-b:pr-only:ungated, job-c:pr-non-draft:gated"), [ + { name: "job-a", rule: "always", gated: true }, + { name: "job-b", rule: "pr-only", gated: false }, + { name: "job-c", rule: "pr-non-draft", gated: true }, + ]) + }) + + it("rejects malformed specifications", () => { + assert.throws(() => parseWatchSpec("job-a:always")) + assert.throws(() => parseWatchSpec("job-a:sometimes:gated")) + assert.throws(() => parseWatchSpec("job-a:always:maybe")) + assert.throws(() => parseWatchSpec("")) + }) +}) + +describe("expected job outcomes", () => { + const base = { eventName: "pull_request", prDraft: false, docsOnly: false } + + it("requires success for always-running jobs on pull requests", () => { + assert.equal(expectedResult({ rule: "always", gated: true }, base), "success") + }) + + it("skips gated jobs only for documentation-only pull request classifications", () => { + assert.equal(expectedResult({ rule: "always", gated: true }, { ...base, docsOnly: true }), "skipped") + }) + + it("never skips on the merge queue even when a docs-only classification is recorded", () => { + assert.equal( + expectedResult({ rule: "always", gated: true }, { ...base, eventName: "merge_group", docsOnly: true }), + "success", + ) + }) + + it("expect pull-request-only jobs to be skipped on other events", () => { + assert.equal(expectedResult({ rule: "pr-only", gated: false }, { ...base, eventName: "push" }), "skipped") + assert.equal(expectedResult({ rule: "pr-only", gated: true }, { ...base, eventName: "merge_group" }), "skipped") + }) + + it("expect non-draft mutations to be skipped for drafts but not for the merge queue", () => { + assert.equal(expectedResult({ rule: "pr-non-draft", gated: true }, { ...base, prDraft: true }), "skipped") + assert.equal( + expectedResult({ rule: "pr-non-draft", gated: true }, { ...base, eventName: "merge_group" }), + "success", + ) + }) +}) + +describe("gate evaluation", () => { + const watched = [ + { name: "cheap", rule: "always", gated: false }, + { name: "expensive", rule: "always", gated: true }, + ] + + for (const result of ["skipped", "cancelled", "queued", undefined]) { + it(`rejects classification result ${result}`, () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }, { result }), + watched, + eventName: "pull_request", + }) + assert.equal(evaluation.ok, false) + assert.match( + evaluation.violations.join("; "), + result === undefined + ? /result is missing/ + : result === "queued" + ? /non-terminal result/ + : /must either succeed or fail/, + ) + }) + } + it("does not trust docs-only outputs leaked by a failed classifier", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { result: "failure", outputs: { category: "docs-only" } }, + ), + watched, + eventName: "pull_request", + }) + assert.equal(evaluation.ok, false) + assert.equal(evaluation.docsOnly, false) + }) + it("passes when full validation ran for a code change", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(evaluation.ok, evaluation.violations.join("; ")) + }) + + it("passes when gated jobs skip a documentation-only change", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { + result: "success", + outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" }, + }, + ), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(evaluation.ok, evaluation.violations.join("; ")) + assert.ok(evaluation.rows.find((row) => row.name === "expensive").explanation.includes("Documentation-only")) + }) + + it("rejects a docs-only result when validation policy changed despite a matching replay", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { result: "success", outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" } }, + ), + watched, + eventName: "pull_request", + policyChanged: true, + classifierReplay: { + status: 0, + classification: { category: "docs-only", reasonCode: "allowlisted-documentation" }, + }, + }) + assert.equal(evaluation.ok, false) + assert.ok(evaluation.violations.some((violation) => violation.includes("Validation policy changed"))) + }) + + it("accepts policy changes when full validation ran", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }), + watched, + eventName: "pull_request", + policyChanged: true, + }) + assert.equal(evaluation.ok, true) + }) + + it("rejects a gated job that ran despite a docs-only classification", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "success" }, + { + result: "success", + outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" }, + }, + ), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("expected it to be skipped"))) + }) + + it("rejects missing or failed required validation", () => { + for (const actual of ["skipped", "failure", "cancelled"]) { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: actual }), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok, `actual=${actual} must violate the gate`) + } + }) + + it("rejects a missing classification job", () => { + const evaluation = evaluateGate({ + needs: { cheap: { result: "success" }, expensive: { result: "success" } }, + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("missing"))) + }) + + it("rejects a malformed classification category", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "success" }, + { + result: "success", + outputs: { category: "banana", "reason-code": "allowlisted-documentation" }, + }, + ), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("malformed"))) + }) + + it("accepts the fail-safe fallback when classification failed and full validation ran", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }, { result: "failure" }), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(evaluation.ok, evaluation.violations.join("; ")) + assert.ok(evaluation.rows.every((row) => row.explanation.includes("fail-safe"))) + }) + + it("rejects a skipped job after a failed classification", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "skipped" }, { result: "failure" }), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("Required validation missing"))) + }) + + it("validates the classification with a matching deterministic replay", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { + result: "success", + outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" }, + }, + ), + watched, + eventName: "pull_request", + prDraft: false, + classifierReplay: { + status: 0, + classification: { category: "docs-only", reasonCode: "allowlisted-documentation", changedFiles: "1" }, + }, + }) + assert.ok(evaluation.ok, evaluation.violations.join("; ")) + }) + + for (const classification of [ + { category: "full", reasonCode: "different" }, + { category: "docs-only", reasonCode: "non-documentation-path" }, + ]) { + it(`rejects independent replay mismatch ${JSON.stringify(classification)}`, () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }), + watched, + eventName: "pull_request", + prDraft: false, + classifierReplay: { + status: 0, + classification: classification, + }, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("replay disagrees"))) + }) + } + + it("rejects a replay failure when the classify job claimed success", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success", expensive: "success" }), + watched, + eventName: "pull_request", + prDraft: false, + classifierReplay: { status: 1 }, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("could not reproduce"))) + }) + + it("reports watched jobs that are missing from the needs graph", () => { + const evaluation = evaluateGate({ + needs: needsFor({ cheap: "success" }), + watched, + eventName: "pull_request", + prDraft: false, + }) + assert.ok(!evaluation.ok) + assert.ok(evaluation.violations.some((violation) => violation.includes("expensive"))) + }) +}) + +describe("runner-minute metrics", () => { + it("sums job minutes and reports completion time from injected API pages", () => { + const metrics = collectJobMetrics({ + repository: "Zoo-Code-Org/Zoo-Code", + runId: 42, + fetchJson: (route) => { + if (route.endsWith("/jobs?per_page=100&page=1")) { + return { + jobs: [ + { name: "a", started_at: "2026-01-01T00:00:00Z", completed_at: "2026-01-01T00:10:00Z" }, + { name: "b", started_at: "2026-01-01T00:00:00Z", completed_at: "2026-01-01T00:05:30Z" }, + ], + } + } + if (route === "repos/Zoo-Code-Org/Zoo-Code/actions/runs/42") { + return { started_at: "2026-01-01T00:00:00Z" } + } + throw new Error(`unexpected route: ${route}`) + }, + }) + assert.equal(metrics.totalRunnerMinutes, 15.5) + assert.equal(metrics.completionMinutes, 10) + }) +}) + +describe("summary rendering", () => { + const watched = [ + { name: "cheap", rule: "always", gated: false }, + { name: "expensive", rule: "always", gated: true }, + ] + + it("renders the job outcome table, metrics, and violations", () => { + const evaluation = evaluateGate({ + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { + result: "success", + outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" }, + }, + ), + watched, + eventName: "pull_request", + prDraft: false, + }) + const summary = renderGateSummary({ + ...evaluation, + needs: needsFor( + { cheap: "success", expensive: "skipped" }, + { + result: "success", + outputs: { category: "docs-only", "reason-code": "allowlisted-documentation" }, + }, + ), + eventName: "pull_request", + metrics: { perJob: [{ name: "cheap", runnerMinutes: 1.5 }], totalRunnerMinutes: 1.5, completionMinutes: 4 }, + violations: evaluation.violations, + }) + assert.match(summary, /Validation gate/) + assert.match(summary, /docs-only/) + assert.match(summary, /Total runner minutes/) + assert.match(summary, /expensive/) + }) +}) + +// --- Workflow wiring assertions ------------------------------------------- + +const GATING_TERMS = [ + "!cancelled()", + "needs.classify.result != 'success'", + "needs.classify.outputs.category != 'docs-only'", +] + +const WORKFLOW_EXPECTATIONS = [ + { + file: "code-qa.yml", + hasMergeGroup: true, + jobs: [ + { id: "dependency-review", rule: "pr-only", gated: false }, + { id: "invisible-chars", rule: "always", gated: false }, + { id: "format-check", rule: "always", gated: false }, + { id: "check-translations", rule: "always", gated: true }, + { id: "knip", rule: "always", gated: false }, + { id: "compile", rule: "always", gated: false }, + { id: "build-vsix", rule: "pr-only", gated: true, prOnlyCondition: true }, + { id: "unit-test", rule: "always", gated: false }, + ], + }, + { + file: "e2e.yml", + hasMergeGroup: true, + jobs: [{ id: "e2e-mock", rule: "always", gated: true }], + }, + { + file: "visual-regression.yml", + hasMergeGroup: true, + jobs: [ + { id: "webview-visual", rule: "always", gated: true }, + { id: "theme-fixtures", rule: "always", gated: true }, + { id: "extension-host-visual", rule: "always", gated: true }, + ], + }, + { + file: "mutation-testing.yml", + hasMergeGroup: true, + jobs: [{ id: "mutation-diff", rule: "pr-non-draft", gated: true, prOnlyCondition: false }], + }, + { + file: "codeql.yml", + hasMergeGroup: false, + jobs: [{ id: "analyze", rule: "always", gated: true }], + }, +] + +describe("change-based validation workflow wiring", () => { + for (const expectation of WORKFLOW_EXPECTATIONS) { + it(`validates ${expectation.file}`, () => { + const workflow = parse( + fs.readFileSync(path.join(repositoryRoot, ".github/workflows", expectation.file), "utf8"), + ) + assert.ok(workflow.on.pull_request.types.includes("labeled")) + assert.ok(workflow.on.pull_request.types.includes("unlabeled")) + assert.equal(workflow.on.pull_request.paths, undefined) + assert.equal(workflow.on.pull_request["paths-ignore"], undefined) + assert.equal(Boolean(workflow.on.merge_group), expectation.hasMergeGroup) + const jobs = workflow.jobs + assert.deepEqual( + Object.keys(jobs).sort(), + ["classify", "validation-gate", ...expectation.jobs.map((job) => job.id)].sort(), + ) + const gate = jobs["validation-gate"] + assert.equal(gate.if, "always()") + assert.deepEqual([...gate.needs].sort(), ["classify", ...expectation.jobs.map((job) => job.id)].sort()) + const step = gate.steps.find((step) => step.name === "Run validation gate") + assert.equal( + step.env.WATCHED_JOBS, + expectation.jobs.map((job) => `${job.id}:${job.rule}:${job.gated ? "gated" : "ungated"}`).join(","), + ) + assert.equal(step.env.NEEDS_JSON, "${{ toJSON(needs) }}") + assert.equal(step.env.EVENT_NAME, "${{ github.event_name }}") + assert.match(step.env.FORCE_FULL_LABEL, /ci:full-validation/) + assert.match(step.env.FORCE_FULL_VARIABLE, /ZOO_CODE_REQUIRE_FULL_VALIDATION/) + const classify = jobs.classify + const classifyStep = classify.steps.find((step) => step.id === "classify") + assert.equal(classify.steps[0].with["fetch-depth"], 0) + assert.equal(classifyStep.uses, "./.github/actions/classify-changes") + assert.equal(classifyStep.with["event-name"], "${{ github.event_name }}") + assert.equal(classifyStep.with["head-sha"], "${{ github.sha }}") + assert.equal(classifyStep.with["force-full-label"], step.env.FORCE_FULL_LABEL) + assert.equal(classifyStep.with["force-full-variable"], step.env.FORCE_FULL_VARIABLE) + assert.equal(gate.permissions.actions, "read") + assert.equal(gate.steps[0].with["fetch-depth"], 0) + assert.match(step.run, /node scripts\/change-gate\.mjs ci/) + for (const job of expectation.jobs) { + if (job.gated) { + assert.equal(jobs[job.id].needs, "classify") + for (const term of GATING_TERMS) assert.ok(jobs[job.id].if.includes(term)) + assert.ok( + jobs[job.id].if.includes( + job.prOnlyCondition + ? "github.event_name == 'pull_request'" + : "github.event_name != 'pull_request'", + ), + ) + } + } + for (const id of ["classify", "validation-gate"]) + assert.equal(jobs[id].name, `${id} (${expectation.file.replace(/\.yml$/, "")})`) + }) + } + it("keeps required compile and matrix checks while gating expensive steps", () => { + const { jobs } = parse(fs.readFileSync(path.join(repositoryRoot, ".github/workflows/code-qa.yml"), "utf8")) + assert.equal(jobs.compile.name, undefined) + assert.equal(jobs["unit-test"].name, "platform-unit-test (${{ matrix.name }})") + assert.deepEqual(jobs.compile.needs, ["classify", "format-check"]) + assert.equal(jobs.compile.steps[0].env.FORMAT_RESULT, "${{ needs.format-check.result }}") + assert.equal(jobs.compile.steps[0].run, 'test "$FORMAT_RESULT" = success') + assert.ok(jobs["format-check"].steps.some((step) => step.run === "pnpm format:check")) + assert.ok(jobs["format-check"].steps.some((step) => step.run === "pnpm test:change-gate")) + for (const id of ["compile", "unit-test"]) { + assert.equal(jobs[id].if, "${{ !cancelled() }}") + assert.equal( + jobs[id].env.RUN_VALIDATION, + "${{ needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only' }}", + ) + for (const step of jobs[id].steps.slice(1)) + assert.ok(step.if.includes("env.RUN_VALIDATION == 'true'"), step.name) + } + assert.equal(jobs["unit-test"].steps[0].if, "env.RUN_VALIDATION == 'false'") + assert.match(jobs["unit-test"].steps[0].run, /GITHUB_STEP_SUMMARY/) + assert.deepEqual( + jobs["unit-test"].strategy.matrix.include.map((entry) => entry.name), + ["ubuntu-latest", "windows-latest"], + ) + }) +}) + +describe("classifier stdout", () => { + it("parses valid output and rejects malformed output", () => { + assert.deepEqual(parseClassifierStdout("Change classification: full (override-label), 12 changed file(s)"), { + category: "full", + reasonCode: "override-label", + changedFiles: 12, + }) + for (const value of [ + undefined, + "", + "Change classification: full", + "Change classification: full (reason), NaN changed file(s)", + ]) + assert.equal(parseClassifierStdout(value), undefined) + }) +}) + +it("collects all metrics pages", () => { + const routes = [] + const metrics = collectJobMetrics({ + repository: "org/repo", + runId: "123", + fetchJson: (route) => { + routes.push(route) + if (!route.includes("/jobs?")) return { started_at: "2026-01-01T00:00:00Z" } + return { + jobs: Array.from({ length: route.endsWith("page=1") ? 100 : 1 }, (_, i) => ({ + name: `job-${i}`, + started_at: "2026-01-01T00:00:00Z", + completed_at: "2026-01-01T00:01:00Z", + })), + } + }, + }) + assert.deepEqual(routes, [ + "repos/org/repo/actions/runs/123", + "repos/org/repo/actions/runs/123/jobs?per_page=100&page=1", + "repos/org/repo/actions/runs/123/jobs?per_page=100&page=2", + ]) + assert.equal(metrics.perJob.length, 101) + assert.equal(metrics.totalRunnerMinutes, 101) +}) diff --git a/scripts/code-qa-workflow.test.mjs b/scripts/code-qa-workflow.test.mjs index 5e1e1a59c0..0018de8fbd 100644 --- a/scripts/code-qa-workflow.test.mjs +++ b/scripts/code-qa-workflow.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict" +import { parse } from "yaml" import fs from "node:fs" import path from "node:path" import { describe, it } from "node:test" @@ -9,35 +10,42 @@ const workflow = fs.readFileSync(path.join(repositoryRoot, ".github/workflows/co const extensionTurbo = JSON.parse(fs.readFileSync(path.join(repositoryRoot, "src/turbo.json"), "utf8")) const coreTurbo = JSON.parse(fs.readFileSync(path.join(repositoryRoot, "packages/core/turbo.json"), "utf8")) +const workflowDocument = parse(workflow) const workflowStep = (name) => { - const match = workflow.match(new RegExp(`- name: ${name}\\n(?(?:\\s{14,}.*\\n?)*)`)) - assert.ok(match?.groups?.body, `missing workflow step: ${name}`) - return match.groups.body + const step = Object.values(workflowDocument.jobs) + .flatMap((job) => job.steps ?? []) + .find((step) => step.name === name) + assert.ok(step, `missing workflow step: ${name}`) + return step } - const parseWorkflowStep = (name) => { - const body = workflowStep(name) - const field = (key) => { - const line = body.split("\n").find((line) => line.trimStart().startsWith(`${key}:`)) - assert.ok(line, `missing ${key} field in workflow step: ${name}`) - return line.slice(line.indexOf(":") + 1).trim() - } - return { if: field("if"), run: field("run") } + const step = workflowStep(name) + return { if: step.if, run: step.run } } describe("platform unit-test workflow", () => { it("keeps coverage authoritative on Ubuntu and runs equivalent uninstrumented Windows tests", () => { - assert.match(workflow, /name: ubuntu-latest[\s\S]*?collect-coverage: true/) - assert.match(workflow, /name: windows-latest[\s\S]*?collect-coverage: false/) + assert.equal( + workflowDocument.jobs["unit-test"].strategy.matrix.include.find((entry) => entry.name === "ubuntu-latest")[ + "collect-coverage" + ], + true, + ) + assert.equal( + workflowDocument.jobs["unit-test"].strategy.matrix.include.find((entry) => entry.name === "windows-latest")[ + "collect-coverage" + ], + false, + ) assert.ok(!workflow.includes("matrix.upload-coverage")) assert.deepEqual(parseWorkflowStep("Run extension coverage lanes"), { - if: "matrix.collect-coverage", + if: "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}", run: 'pnpm turbo run test:coverage:api test:coverage:core test:coverage:services test:coverage:misc test:coverage:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only', }) assert.deepEqual(parseWorkflowStep("Run extension test lanes"), { - if: "${{ !matrix.collect-coverage }}", + if: "${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}", run: 'pnpm turbo run test:api test:core test:services test:misc test:tree-sitter --filter="zoo-code" --concurrency=2 --log-order grouped --output-logs new-only', }) @@ -62,16 +70,16 @@ describe("platform unit-test workflow", () => { ], ]) { const coverageStep = parseWorkflowStep(coverageStepName) - assert.equal(coverageStep.if, "matrix.collect-coverage") + assert.equal(coverageStep.if, "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}") assert.ok(coverageStep.run.includes(coverageCommand), `missing command in step: ${coverageStepName}`) const plainStep = parseWorkflowStep(plainStepName) - assert.equal(plainStep.if, "${{ !matrix.collect-coverage }}") + assert.equal(plainStep.if, "${{ env.RUN_VALIDATION == 'true' && (!matrix.collect-coverage) }}") assert.ok(plainStep.run.includes(plainCommand), `missing command in step: ${plainStepName}`) assert.ok(!plainStep.run.includes("--coverage"), `${plainStepName} must not collect coverage on Windows`) } - assert.ok(!workflowStep("Run extension dist smoke test").includes("if:")) + assert.equal(parseWorkflowStep("Run extension dist smoke test").if, "env.RUN_VALIDATION == 'true'") }) it("does not run coverage verification or uploads on Windows", () => { @@ -86,7 +94,7 @@ describe("platform unit-test workflow", () => { "Upload core integration coverage to Codecov", "Upload coverage reports to GitHub", ]) { - assert.match(workflowStep(stepName), /if: matrix\.collect-coverage/) + assert.equal(workflowStep(stepName).if, "${{ env.RUN_VALIDATION == 'true' && (matrix.collect-coverage) }}") } }) @@ -128,23 +136,22 @@ describe("Knip quality gate", () => { ["Run knip checks", "pnpm knip"], ["Run production Knip checks", "pnpm knip:production"], ]) { - assert.match(workflowStep(stepName), new RegExp(`run: ${command}\\s*$`)) - assert.doesNotMatch(workflowStep(stepName), /continue-on-error|if:/) + const step = workflowStep(stepName) + assert.equal(step.run, command) + assert.equal(step.if, undefined) + assert.equal(step["continue-on-error"], undefined) } - const job = workflow.match(/\n knip:\n(?[\s\S]*?)\n compile:/)?.groups?.body - assert.ok(job, "missing knip job") - assert.doesNotMatch(job, /continue-on-error|if:/) + assert.equal(workflowDocument.jobs.knip.if, undefined) + assert.equal(workflowDocument.jobs.knip["continue-on-error"], undefined) }) }) describe("Code QA dependency install", () => { it("installs every job with a frozen lockfile", () => { - const setups = workflow.match(/uses: \.\/\.github\/actions\/setup-node-pnpm\n/g) ?? [] - const frozen = - workflow.match( - /uses: \.\/\.github\/actions\/setup-node-pnpm\n\s+with:\n\s+install-args: "--frozen-lockfile"\n/g, - ) ?? [] + const setups = Object.values(workflowDocument.jobs) + .flatMap((job) => job.steps ?? []) + .filter((step) => step.uses === "./.github/actions/setup-node-pnpm") assert.ok(setups.length > 0, "missing setup-node-pnpm steps") - assert.equal(frozen.length, setups.length) + for (const step of setups) assert.equal(step.with["install-args"], "--frozen-lockfile") }) }) diff --git a/scripts/stryker-diff.test.mjs b/scripts/stryker-diff.test.mjs index 339e44e676..abf96eab65 100644 --- a/scripts/stryker-diff.test.mjs +++ b/scripts/stryker-diff.test.mjs @@ -81,9 +81,15 @@ describe("mutation testing workflow", () => { it("waits until a draft pull request is ready before emitting mutation annotations", () => { const workflow = readWorkflow() - assert.ok(workflow.includes("types: [edited, opened, reopened, ready_for_review, synchronize]")) + assert.ok(parseYaml(workflow).on.pull_request.types.includes("ready_for_review")) + // The job condition keeps the draft and merge-group semantics asserted + // below and adds the fail-safe change-classification gate from issue + // #2291: only a successful documentation-only pull-request + // classification may skip the job. assert.ok( - workflow.includes("if: github.event_name == 'merge_group' || github.event.pull_request.draft == false"), + workflow.includes( + "if: ${{ !cancelled() && (github.event_name == 'merge_group' || github.event.pull_request.draft == false) && (needs.classify.result != 'success' || github.event_name != 'pull_request' || needs.classify.outputs.category != 'docs-only') }}", + ), ) const draftToReadyRuns = [