diff --git a/.changeset/happy-lands-sink.md b/.changeset/happy-lands-sink.md new file mode 100644 index 0000000000..853d812bb3 --- /dev/null +++ b/.changeset/happy-lands-sink.md @@ -0,0 +1,3 @@ +--- + +--- diff --git a/.github/workflows/pr-bundle-size-comment.yml b/.github/workflows/pr-bundle-size-comment.yml index 349e1559d7..3449b96caf 100644 --- a/.github/workflows/pr-bundle-size-comment.yml +++ b/.github/workflows/pr-bundle-size-comment.yml @@ -31,7 +31,22 @@ jobs: with: node-version: '22' + - name: Find bundle-size artifact + id: artifact + env: + GH_TOKEN: ${{ github.token }} + SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + shell: bash + run: | + set -euo pipefail + found=$(gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts?per_page=100" --jq '[.[].artifacts[]] | any(.name == "bundle-size-analysis" and .expired == false)') + echo "found=$found" >> "$GITHUB_OUTPUT" + if [[ "$found" != "true" ]]; then + echo "No bundle-size artifact for this attempt; report-only verification runs do not rebuild bundles." >> "$GITHUB_STEP_SUMMARY" + fi + - name: Download bundle-size report + if: steps.artifact.outputs.found == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: bundle-size-analysis @@ -41,9 +56,11 @@ jobs: run-id: ${{ github.event.workflow_run.id }} - name: Validate downloaded bundle size report + if: steps.artifact.outputs.found == 'true' run: node apps/bundle-size/scripts/validate-report.mts bundle-size-analysis/report.md - name: Publish bundle size PR comment + if: steps.artifact.outputs.found == 'true' env: EXPECTED_HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} EXPECTED_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/pr-control-assessment.yml b/.github/workflows/pr-control-assessment.yml new file mode 100644 index 0000000000..19208d7a72 --- /dev/null +++ b/.github/workflows/pr-control-assessment.yml @@ -0,0 +1,82 @@ +name: Control Assessment PR Summary + +on: + workflow_run: + workflows: [PR] + types: [completed] + +permissions: + actions: read + contents: read + pull-requests: read + +concurrency: + group: control-assessment-${{ github.event.workflow_run.head_repository.id }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + publish: + name: Publish trusted component assessment + if: ${{ github.event.workflow_run.event == 'pull_request' && vars.CONTROL_ASSESSMENT_PUBLISH == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout trusted publisher + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + - name: Find source assessment + id: artifact + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/src/ci/controlAssessmentCiTask.ts find-artifact + + - name: Download source assessment data + if: steps.artifact.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.artifact.outputs.artifact-name }} + path: ${{ runner.temp }}/component-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.event.workflow_run.id }} + + # Install the formatter from trusted workflow configuration, never from PR code. + - name: Set up trusted report formatter + if: steps.artifact.outputs.found == 'true' + run: | + npm install --prefix "$RUNNER_TEMP/assessment-tools" --ignore-scripts --no-audit --no-fund oxfmt@0.61.0 + echo "FURN_ASSESSMENT_TOOLS=$RUNNER_TEMP/assessment-tools" >> "$GITHUB_ENV" + + - name: Validate publication without write credentials + id: prepare + if: steps.artifact.outputs.found == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + BUNDLE_DIRECTORY: ${{ runner.temp }}/component-source-evidence + run: node scripts/src/ci/controlAssessmentCiTask.ts prepare-publication + + - name: Create narrowly scoped publisher token + id: app-token + if: steps.prepare.outputs.eligible == 'true' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + with: + client-id: ${{ vars.CONTROL_ASSESSMENT_APP_CLIENT_ID }} + private-key: ${{ secrets.CONTROL_ASSESSMENT_APP_PRIVATE_KEY }} + permission-contents: write + + - name: Commit the generated summary and SPEC assessment fields to the PR branch + if: steps.prepare.outputs.eligible == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + ASSESSMENT_WRITE_TOKEN: ${{ steps.app-token.outputs.token }} + BUNDLE_DIRECTORY: ${{ runner.temp }}/component-source-evidence + run: node scripts/src/ci/controlAssessmentCiTask.ts publish diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3117b5b6eb..3bddc70466 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,10 +13,91 @@ concurrency: cancel-in-progress: true jobs: + # A separate optional publisher can add the generated report and SPEC assessment fields + # to this PR. Verify those report-only follow-ups before skipping source checks. + assessment-inputs: + name: Validate assessment inputs + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + # Pass the verified decision and original source result to the other jobs. + # Unless report-only reuse is verified, take the normal source-check path. + outputs: + report-only: ${{ steps.verify.outputs.report-only || steps.inspect.outputs.report-only || 'false' }} + source-result: ${{ steps.verify.outputs.source-result }} + steps: + # Use the validator from the target branch, not rules proposed by the PR. + # Fetch its history without leaving checkout credentials for later steps. + - name: Checkout trusted input validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + # Until the validator lands on the base branch, run all normal source checks. + # An inspection error must fail this job, not silently approve report-only reuse. + - name: Inspect report-only candidate + id: inspect + env: + GITHUB_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + if [[ -f scripts/src/ci/controlAssessmentCiTask.ts ]]; then + node scripts/src/ci/controlAssessmentCiTask.ts inspect + else + echo "candidate=false" >> "$GITHUB_OUTPUT" + echo "report-only=false" >> "$GITHUB_OUTPUT" + echo "Trusted assessment validator is not on the base branch yet. Running all normal source checks." >> "$GITHUB_STEP_SUMMARY" + fi + + # Get evidence from the original run referenced by this candidate, not another run. + - name: Download original assessment evidence + if: steps.inspect.outputs.candidate == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.inspect.outputs.artifact-name }} + path: ${{ runner.temp }}/component-source-evidence + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ steps.inspect.outputs.run-id }} + + # Looking report-only is not enough: verify unchanged tested inputs and the exact + # generated report/SPEC assessment fields before reusing the saved source result. + # Use only the pinned formatter, without lifecycle scripts or PR dependencies. + - name: Set up trusted report formatter + if: steps.inspect.outputs.candidate == 'true' + run: | + npm install --prefix "$RUNNER_TEMP/assessment-tools" --ignore-scripts --no-audit --no-fund oxfmt@0.61.0 + echo "FURN_ASSESSMENT_TOOLS=$RUNNER_TEMP/assessment-tools" >> "$GITHUB_ENV" + + - name: Verify unchanged inputs and generated report plus SPEC assessment fields + id: verify + if: steps.inspect.outputs.candidate == 'true' + env: + CANDIDATE_PATH: ${{ steps.inspect.outputs.candidate-path }} + BUNDLE_DIRECTORY: ${{ runner.temp }}/component-source-evidence + run: node scripts/src/ci/controlAssessmentCiTask.ts verify + + # The source jobs below wait for that decision. Normal code changes still run them; + # only verified report-only follow-ups skip the repeated builds and tests. js: name: JS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 + # Tell the assessor whether this job actually saved an evidence artifact. + outputs: + unit-artifact: ${{ steps.unit-evidence.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -34,11 +115,33 @@ jobs: - name: Test agent permissions run: yarn test:agent-permissions + # Record the inputs before testing so results can be tied to the tested version. + - name: Capture existing component test inputs + run: node scripts/src/ci/controlAssessmentCiTask.ts begin-unit + + # Keep the existing build and tests; capture their component Jest results for the + # assessor rather than launching a second unit-test run. - name: Build CI + env: + FURN_ASSESSMENT_UNIT_CAPTURE: ${{ github.workspace }}/apps/storybook/artifacts/assessment-unit-ci run: yarn lage buildci + # Save captured evidence after failures too, unless cancelled. Attempt-scoped + # names keep reruns separate; missing files warn, but do not count as passing tests. + - name: Upload existing component unit evidence + id: unit-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: component-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + if-no-files-found: warn + retention-days: 14 + bundle-size: name: Bundle Size PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 30 permissions: @@ -108,6 +211,8 @@ jobs: android: name: Android PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -132,6 +237,8 @@ jobs: macos: name: macOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -180,6 +287,8 @@ jobs: macos-storybook: name: macOS Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -233,6 +342,8 @@ jobs: macos-paper-storybook: name: macOS Paper Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 40 steps: @@ -284,6 +395,8 @@ jobs: ios: name: iOS PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: macos-26 timeout-minutes: 60 steps: @@ -321,6 +434,8 @@ jobs: windows: name: Windows PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -397,6 +512,8 @@ jobs: windows-storybook: name: Windows Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -463,6 +580,8 @@ jobs: win32: name: Win32 PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 steps: @@ -511,17 +630,26 @@ jobs: win32-storybook: name: Win32 Storybook PR + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: windows-latest timeout-minutes: 60 + # Tell the assessor whether this job actually saved a native evidence artifact. + outputs: + native-artifact: ${{ steps.native-evidence.outputs.artifact-id }} steps: + # Full history lets evidence capture identify the tested merge and source parents. - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + # Use Node 24 for the native TypeScript evidence-collection scripts. - name: Set up toolchain uses: microsoft/react-native-test-app/.github/actions/setup-toolchain@6046bdd4914c7ceb35ec83b92ae3211a03ddccef # 5.4.14 with: platform: windows - node-version: 22 + node-version: 24 - name: Install dependencies run: yarn @@ -529,6 +657,8 @@ jobs: - name: Build packages run: yarn build + # This pilot keeps keyboard, physical clicks and wheel input disabled. + # Tests that require them must still appear as evidence gaps, not passes. - name: Set up Win32 Desktop Driver id: desktop-driver uses: ./.github/actions/setup-desktop-driver @@ -536,10 +666,38 @@ jobs: platform: win32 disabled-input-features: keyboard,physicalClick,wheel + # Record source and assessment inputs before running the existing Storybook tests. + - name: Capture configured component execution inputs + id: assessment-inputs + run: node scripts/src/ci/controlAssessmentCiTask.ts begin + + # Run only if input capture succeeded and this run was not cancelled. + # Keep the step ID so evidence collection can record the smoke outcome. - name: Smoke test Win32 Storybook + id: smoke + if: ${{ !cancelled() && steps.assessment-inputs.outcome == 'success' }} run: yarn storybook smoke --win32 --mode stories-and-tests working-directory: apps/storybook + # Collect available reports and the smoke outcome even after a failure, unless cancelled. + - name: Collect configured component native evidence + if: ${{ always() && !cancelled() }} + env: + SMOKE_OUTCOME: ${{ steps.smoke.outcome }} + run: node scripts/src/ci/controlAssessmentCiTask.ts finish + + # Preserve this attempt's native evidence after failures too. Unlike the unit + # upload, missing native evidence files fail this step. + - name: Upload component native evidence + id: native-evidence + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: component-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + - name: Upload Win32 Storybook artifacts if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -553,11 +711,75 @@ jobs: ${{ steps.desktop-driver.outputs.cache-root }}/v1/diagnostics/** if-no-files-found: warn + # Wait for the evidence jobs, but still assess what is available if they failed. + # Skip cancellations, invalid assessment inputs and verified report-only follow-ups. + # This advisory job reads existing results; it does not rerun tests or assign a stage. + control-assessment: + name: Assess component evidence + needs: [assessment-inputs, js, win32-storybook] + if: ${{ always() && !cancelled() && needs.assessment-inputs.result == 'success' && needs.assessment-inputs.outputs.report-only != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # Assess the PR's tested merge and its history, without leaving checkout credentials. + - name: Checkout assessed source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + + # Provide the package command and Lage, without running component tests again. + - name: Install assessment dependencies + run: node .yarn/releases/yarn-4.18.0.cjs install --immutable + + # Download this attempt's Win32 evidence only if the job saved an artifact. + - name: Download native evidence + if: needs.win32-storybook.outputs.native-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: component-native-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + + # Do the same for component-test results. Missing artifacts remain evidence gaps. + - name: Download existing unit evidence + if: needs.js.outputs.unit-artifact != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: component-unit-evidence-${{ github.run_attempt }} + path: apps/storybook/artifacts/assessment-unit-ci + + # Combine the saved results and both jobs' outcomes into coverage and gap findings. + # The readable report goes into the job summary; this does not rerun component tests. + - name: Assess existing results without rerunning tests + env: + UNIT_OUTCOME: ${{ needs.js.result }} + NATIVE_OUTCOME: ${{ needs.win32-storybook.result }} + FURN_ASSESSMENT_CI: 'true' + run: node .yarn/releases/yarn-4.18.0.cjs lage assess-components + + # Keep per-control reports for 14 days, even after failures unless cancelled. + # Missing report files fail the upload rather than silently losing the assessment. + - name: Upload component assessments + if: ${{ always() && !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: component-assessments-${{ github.run_attempt }} + path: apps/storybook/artifacts/win32/assessment-ci + if-no-files-found: error + retention-days: 14 + check-changesets: name: Check for Changesets + needs: assessment-inputs runs-on: ubuntu-latest - # Skip this check for the version bump PR created by changesets - if: ${{ github.head_ref != 'changeset-release/main' }} + # Keep the release-PR exception and also skip verified report-only follow-ups: + # adding generated assessment metadata is not another product change needing release notes. + if: ${{ needs.assessment-inputs.outputs.report-only != 'true' && github.head_ref != 'changeset-release/main' }} steps: - name: Checkout @@ -583,6 +805,8 @@ jobs: publish-dry-run-pack: name: NPM Publish Dry Run — Pack + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 outputs: @@ -624,6 +848,8 @@ jobs: test-links: name: Test repo links + needs: assessment-inputs + if: needs.assessment-inputs.outputs.report-only != 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -641,11 +867,13 @@ jobs: - name: Test markdown links run: yarn lage test-links --verbose --grouped - pr: - name: PR + # Keep "did the source checks pass?" separate from optional assessment publication. + pr-source: + name: PR source validation if: always() runs-on: ubuntu-latest needs: + - assessment-inputs - js - android - macos @@ -656,6 +884,12 @@ jobs: - publish-dry-run-pack - test-links steps: + # A report-only commit cannot erase a failed original source-validation result. + - name: Preserve reused source result + if: ${{ needs.assessment-inputs.outputs.report-only == 'true' && needs.assessment-inputs.outputs.source-result != 'success' }} + run: | + echo "The original source validation did not pass. A report-only commit cannot turn it green." + exit 1 - name: Check for failures or cancellations if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} run: | @@ -663,3 +897,29 @@ jobs: exit 1 - name: All required jobs passed run: echo "All required jobs completed successfully." + + # Keep the existing required status named PR. It checks source validation and, + # when enabled, the verified same-PR assessment-publication follow-up. + pr: + name: PR + if: always() + runs-on: ubuntu-latest + needs: [assessment-inputs, pr-source] + steps: + # Both the initial input validation and source-validation result must pass. + - name: Require successful source validation + if: ${{ needs.assessment-inputs.result != 'success' || needs.pr-source.result != 'success' }} + run: | + echo "Source validation or assessment-input validation failed." + exit 1 + # Only same-repository PRs into the default branch participate when publishing is on. + # Fail this source run rather than wait: the separate trusted publisher must add the + # report, and the report-only follow-up must verify it before this check can pass. + - name: Require same-PR assessment when publishing is enabled + if: ${{ vars.CONTROL_ASSESSMENT_PUBLISH == 'true' && github.event.pull_request.head.repo.full_name == github.repository && github.base_ref == github.event.repository.default_branch && needs.assessment-inputs.outputs.report-only != 'true' }} + run: | + echo "Source CI completed. Waiting for the trusted publisher to commit the component assessment to this PR." + echo "The report-only follow-up run must verify that report before this required check can pass." + exit 1 + - name: Required PR validation passed + run: echo "Source checks and the applicable assessment-publication requirements passed." diff --git a/lage.config.mjs b/lage.config.mjs index 563ec24fe8..4fb48282cf 100644 --- a/lage.config.mjs +++ b/lage.config.mjs @@ -4,6 +4,9 @@ import path from 'node:path'; const config = { npmClient: 'yarn', pipeline: { + 'assess-components': { + cache: false, + }, // ── Per-package tasks ────────────────────────────────────────────────── bundle: { inputs: ['**/*', '!node_modules/**/*', '!dist/**/*', '!lib/**/*', '!lib-commonjs/**/*'], diff --git a/packages/agentic/components/README.md b/packages/agentic/components/README.md index a727661c04..0bff6adc47 100644 --- a/packages/agentic/components/README.md +++ b/packages/agentic/components/README.md @@ -41,6 +41,7 @@ Each component's `src/components//SPEC.md` describes its supported props, | [Tests and stories](references/tests-and-stories.md) | Runtime, type, and Storybook coverage, including native story tests | | [Contributing](references/contributing.md) | Source ownership, test commands, and validation | | [Remaining work](references/remaining-work.md) | Open focus qualification, missing components, and contract decisions | +| [Component assessment](references/assessments/README.md) | Advisory assessment commands, mappings, evidence and CI | The [reference index](references/README.md) lists every reference by ID. Agents working in this package start from [AGENTS.md](AGENTS.md). diff --git a/packages/agentic/components/package.json b/packages/agentic/components/package.json index 0615974c1a..da2a411dc6 100644 --- a/packages/agentic/components/package.json +++ b/packages/agentic/components/package.json @@ -21,6 +21,7 @@ "spec-source-lock.json", "spec-source-report.json", "src/", + "!src/components/*/assessment/**", "tsconfig.json", "tsconfig.stories.json" ], @@ -44,6 +45,7 @@ } }, "scripts": { + "assess-components": "fluentui-scripts assess-components", "build": "tsc -b", "check:spec-contracts": "node ./scripts/spec-contracts/check-spec-contracts.cjs", "clean": "fluentui-scripts clean", @@ -105,7 +107,10 @@ } }, "furn": { - "jestPlatform": "macos" + "jestPlatform": "macos", + "assessments": [ + "src/components/checkbox/assessment.mapping.json" + ] }, "rnx-kit": { "kitType": "library", diff --git a/packages/agentic/components/references/README.md b/packages/agentic/components/references/README.md index 3abce851b0..5bd41614d4 100644 --- a/packages/agentic/components/references/README.md +++ b/packages/agentic/components/references/README.md @@ -26,3 +26,5 @@ owner_paths: | `components.agentic.tests-and-stories` | [Tests and stories](tests-and-stories.md) | Tests, stories, `wdio` story tests, or validation | | `components.contributing` | [Contributing](contributing.md) | Finding source ownership, test commands, or validation | | `components.remaining-work` | [Remaining work](remaining-work.md) | Planning focus qualification, missing components, or open contract decisions | +| `components.assessments` | [Component assessment](assessments/README.md) | Running assessment, configuring mappings, or understanding CI and publication | +| `components.assessments.checkbox` | [Checkbox assessment](assessments/checkbox.md) | Reviewing Checkbox evidence, coverage, or mapping drift | diff --git a/packages/agentic/components/references/assessments/README.md b/packages/agentic/components/references/assessments/README.md new file mode 100644 index 0000000000..43a909918f --- /dev/null +++ b/packages/agentic/components/references/assessments/README.md @@ -0,0 +1,220 @@ +--- +id: components.assessments +kind: instruction-reference +owner_paths: + - packages/agentic/components +--- + +# Advisory component assessment + +Assessment explains what existing test results establish against reviewed requirements, and what remains unproven. +It is not a new test framework, release gate, or automatic maturity decision. A successful command means that reports +were generated, not that a control is ready. Lifecycle metadata and Backlog/Experimental/Beta/Production decisions +are not changed. + +The shared implementation lives in `@fluentui-react-native/scripts/src/ci/assessment/`; its workflow entrypoint is +`@fluentui-react-native/scripts/src/ci/controlAssessmentCiTask.ts`. Normal developer commands remain in +`@fluentui-react-native/scripts/src/tasks/assessControlTask.ts`. This package owns its control definitions and guidance. +See [Checkbox](checkbox.md) for the pilot's reviewed coverage and [remaining work](../remaining-work.md) for deferred work. + +## Package command and discovery + +From the repository root: + +```powershell +yarn lage assess-components +``` + +Lage invokes each package that declares `assess-components`. This package delegates to `fluentui-scripts assess-components`. +The task is deliberately uncached: execution evidence, freshness, and the current workflow attempt must be checked each time. +Assessment is not part of `buildci`, because its final CI execution must wait for the existing native and JS jobs. + +To list this package's configured and unconfigured controls without assessing them: + +```powershell +yarn workspace @fluentui-react-native/components assess-components --list +``` + +The command discovers the package's `furn.assessments` entries, rather than assuming a particular control name. +Each entry points to one `src/components//assessment.mapping.json`, beside that control's `SPEC.md`. +Controls with a SPEC but no configured mapping are reported as `not-configured`, never passed or implicitly mapped. +An explicitly requested unconfigured control fails. Ordinary PRs currently assess all configured controls, not only +ones affected by the diff. + +Each schema-v2 mapping declares: + +- component ID, display name, and package-relative component directory; +- contract files, assertion-bearing native helpers, reviewed fingerprint, mapping version, exact native callback, + physical-input requirement, and a requirement-first matrix of specific acceptance checks; +- package-relative runtime test files, repository-relative unit fingerprint inputs, declared Jest mock platform, + reviewed unit fingerprint, and an explicitly excluded case inventory; +- control-specific limitations. + +The owning package supplies its workspace name and declared test script. Shared capture and evaluation code does not +contain a control catalog or hard-coded test filenames. Definitions are data, not executable configuration. Invalid +paths, duplicate controls or assertion identities, unknown requirement references, and a conflicting Jest profile fail +explicitly. A filename or a SPEC evidence reference alone never proves requirement coverage. + +## Requirement-first mapping + +The mapping describes **what must be checked and which tests check it**, not execution results: + +```json +{ + "CBX-001": { + "checks": [ + { + "id": "default-unchecked", + "description": "Omitting status and defaultStatus exposes checked=false.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + } + ], + "remaining": "Additional checks and evidence remain necessary." + } +} +``` + +This abbreviated example shows one row, not the complete requirement. Define the checklist from the SPEC and its +companions before inspecting test coverage; do not define requirements by the tests that happen to exist. +Every SPEC requirement must have its complete acceptance-check set, including checks with no tests. Never shrink or +filter that set based on available tests or execution results. Completeness requires reviewing all applicable contract +clauses; merely listing every requirement ID is not a completeness proof. `source` cites the contract file/section from which the check +was derived. `evidence` declares whether the claim needs unit, native, type, or visual verification. +`tests: []` keeps an uncovered check visible; it means no approved test link, not necessarily no existing test. +Type and visual adapters are not implemented, so those checks remain uncovered. + +Unit links identify exact Jest full names in the configured runtime files. Native links identify the exact configured +story/callback. Multiple tests can be linked to one check, and one test can appear under multiple checks. A shared +check ID can appear under several requirements only with an identical definition. Every linked test must qualify +before a check passes. A skipped, missing, stale, unverified or failed test cannot qualify a row. +Execution-case counts deduplicate shared test links; they are not inflated by repeated rows. +`unit.excludedCases` records reviewed tests that receive no target requirement credit and why. + +The pilot prepares this matrix from its reviewed premade mapping before assessing saved evidence. Automatic proposal +generation and owner review remain future mapping-maintenance work. Generated Markdown contains the readable +requirement/check/test matrix with evidence outcomes; generated assessment JSON contains the same rows. +A passing row establishes only that mapped check, not complete parent coverage, native behavior from JS, or maturity. + +## Existing evidence and explicit local execution + +```powershell +yarn workspace @fluentui-react-native/components assess-components --component checkbox --evidence "C:\furn\apps\storybook\artifacts\desktop\\run.json" +yarn workspace @fluentui-react-native/components assess-components --component checkbox --unit-evidence "C:\evidence\jest.json" +yarn workspace @fluentui-react-native/components assess-components --component checkbox --run-unit-tests --max-age-hours 24 +``` + +Evidence and output arguments are relative to the command's working directory; use absolute paths when selecting a +workspace from another directory. Canonical native input is the `desktop-test` supervisor `run.json` with its declared +Storybook companion retained in the same run directory. Historical direct native and smoke reports are accepted for +replay, but absent provenance remains unverified. + +Default execution reads reports only. Omitting reports produces an honest missing-evidence inventory. +`--run-unit-tests` is an explicit local option that uses the package's declared test script, selected runtime files, +and Jest JSON output. It does not substitute type-test Jest results for a compiler check. It cannot be combined with +replayed unit inputs. Each execution gets a fresh UUID-named directory. + +`--context`, `--build-id`, `--target-id`, and a positive `--max-age-hours` are needed to qualify bound native evidence. +Unit replay uses its original `--unit-context`, `--unit-build-id`, and explicit age policy. Context must be created +during execution, not reconstructed later from current files. It binds exact result bytes, source commit, dirty state, +contract/mapping fingerprints, platform, execution interval, and build identity. Native target identity is captured +before execution. A dirty checkout or missing binding keeps observations unverified; native and unit bindings remain separate. + +`--output` chooses an artifact directory. A single control writes `assessment.json`, `assessment.md`, and an exact +`assessment.mapping.json` copy there. The copy contains the reviewed matrix, without execution results. +Batch local assessment uses separate package/control subdirectories. These outputs are ignored by default. +`--update-spec` additionally writes a clearly marked local report and the managed assessment fields in the adjacent +SPEC; it is never a bound CI publication. Input/output aliases, linked SPEC destinations, and concurrent SPEC changes +are rejected. + +## Evidence rules + +Choose evidence that proves the claim; a requirement does not inherently need both a unit and a native test. + +| Claim | Appropriate evidence | +| ------------------------------------------------------------------------ | --------------------------------------------------------- | +| Defaults, state ownership/transitions, callbacks, resolved styles | Complete code-level assertions can suffice. | +| Public TypeScript constraints | A compiler/type-check result, not a Jest types-file pass. | +| Rendered wrapping, layout, focus-ring geometry | Layout or visual execution. | +| Actual platform accessibility actions, focus, keyboard or physical input | Native execution for the integration claim. | + +Preserve failures, skips, missing cases, unconfigured controls, retries, flaky execution, stale mappings, and mismatched +inputs. Enclosing execution failures prevent recorded passes from qualifying. Unmapped test results remain visible +but receive no requirement credit. Multi-assertion failures do not identify which individual assertion is defective. + +The native adapter currently supports Win32/Paper Storybook, and the unit adapter parses Jest JSON. Type/build, +visual/manual ingestion and other native targets are not implemented. Shared configuration does not qualify those targets. + +## CI flow and outputs + +The PR workflow captures inputs before the existing JS package tests and Win32 smoke run. The package Jest invocation +is called exactly once with its existing selection, arguments, platform, and exit status; capture adds JSON output. +One package execution can supply separately bound evidence for multiple configured controls. Other packages and +platform invocations are unchanged. + +JS and native jobs remain parallel. The later **Assess component evidence** job downloads same-attempt artifacts, +installs the tools needed for Lage, and runs the same package command with `FURN_ASSESSMENT_CI=true`. That mode combines +saved evidence only: local replay flags, test execution, and tracked-file writes are prohibited. It still runs after +ordinary source-job failures, but not after cancellation or invalid input validation. + +Source artifacts are `component-unit-evidence-` and `component-native-evidence-`. +Final output is `component-assessments-`, retained for 14 days. Inside it, each configured control has +`//bundle.json`, `assessment.json`, `ASSESSMENT.md`, `assessment.mapping.json`, proposed `SPEC.md`, +and any original unit JSON/context. The mapping is the exact reviewed source used for that execution, not a reconstruction +from passing test names. +The same readable report appears in the job summary. Files are created under ignored +`apps/storybook/artifacts/win32/assessment-ci/`, not committed by the assessment job. + +Each bundle binds repository, PR, source head/base, actual tested merge commit, complete tracked input digest, +workflow run, attempt, and execution times. A native capture accepts exactly one fresh desktop run and verifies its +ID and target; it does not select by modification time. Missing/cached outputs stay missing. A failed source validation +cannot become a successful PR through assessment. Hosted `keyboard`, `physicalClick`, and `wheel` remain disabled. + +## Formatting and exact report verification + +Generated Markdown follows `oxfmt`, including the checked-in report path; there is no assessment-specific exclusion. +Generation, publication, and report-only verification use the same formatter and repository configuration, then compare +the complete expected output exactly. Formatting does not weaken rejection of altered findings, metadata, or SPEC content. +Formatter errors are failures, not permission to accept an unformatted report. + +Normal jobs use repository dependencies. Trusted validation/publication jobs install only the pinned formatter in a +temporary tools directory with lifecycle scripts disabled. They do not install PR dependencies or execute PR code. +Keep that pin aligned with the repository lockfile when upgrading the formatter. + +## Optional same-PR publication + +Publication remains disabled unless a maintainer separately enables it. The default-branch workflow reconstructs +reports from original evidence with trusted code. It never checks out or executes the PR with write credentials. +Publication validates all configured controls together and writes their generated reports, SPEC fields, and reviewed +mapping snapshots in one atomic commit. Report-only reuse requires the complete configured set from the same tested workflow attempt. + +The only allowed writes are that control's generated `assessment/ASSESSMENT.md`, the strict, canonical +`assessment-*` front-matter block in its SPEC, and the exact tested `assessment.mapping.json`. +The publisher independently reads the mapping from the tested Git tree and rejects an altered or missing artifact copy. +In this premade pilot the source mapping already has those bytes, so publishing it normally creates no mapping diff. +It never invents checks, refreshes reviewed fingerprints, or silently changes test links. +Human-authored contract bytes and lifecycle remain unchanged. +The block records package version, source/tested identity, dirty state, Win32/Paper target, outcome, health, provenance, +generation time, report pointer, and an undetermined stage. Only matched clean native evidence supplies the native-tested SHA. +The later report commit is not relabeled as the tested commit. + +The tracked-input digest excludes configured generated summary paths and removes only validated SPEC assessment +blocks; modes and all other source, contract, mapping, dependency, workflow and configuration bytes remain inputs. +Reuse requires an immediate report commit changing exactly the report and SPEC per control, the unchanged reviewed mapping, +unchanged merge-tree inputs, the matching original artifact, +exact reconstructed report/SPEC, and the original **PR source validation** result. The explicit reuse limit is 168 hours. +Old or expired evidence falls back to full CI or fails verification; it never creates current coverage. +Any future mapping proposal that changes checks, links, or fingerprints remains a source-input change and must undergo +review and normal CI, not report-only reuse. + +Activation requires trusted tooling on the default branch, a repository-scoped GitHub App with only Contents write, +`CONTROL_ASSESSMENT_APP_CLIENT_ID`, secret `CONTROL_ASSESSMENT_APP_PRIVATE_KEY`, and +`CONTROL_ASSESSMENT_PUBLISH=true`. A maintainer must verify organization policy and validate the full hosted flow first. +No token or key belongs in source, reports, or logs. Forks and non-default-target PRs retain artifacts without branch writes. + +For eligible opt-in runs, the required **PR** check waits for publication and exact report-only verification. +API failures, changed heads, ambiguous evidence, and invalid generated fields are not successful publication. +The commit API's expected-head check prevents races; there is no force push, merge, approval, or protection bypass. +Disable the opt-in and rerun CI to return to artifact-only operation. Resolve real merge conflicts before regenerating +reports; do not merge assessment prose as if it were current execution evidence. diff --git a/packages/agentic/components/references/assessments/checkbox.md b/packages/agentic/components/references/assessments/checkbox.md new file mode 100644 index 0000000000..6865255528 --- /dev/null +++ b/packages/agentic/components/references/assessments/checkbox.md @@ -0,0 +1,82 @@ +--- +id: components.assessments.checkbox +kind: instruction-reference +owner_paths: + - packages/agentic/components/src/components/checkbox +--- + +# Checkbox assessment pilot + +Checkbox is the only configured control in the pilot. The reusable assessment command, discovery, evidence capture, +and evaluation are described in [Advisory component assessment](README.md). Nothing in this pilot assigns or promotes maturity. + +## Owned files + +- Contract: `src/components/checkbox/SPEC.md` and its `spec/` companions. +- Reviewed machine mapping: `src/components/checkbox/assessment.mapping.json`, beside the SPEC. +- Existing runtime tests: `src/components/checkbox/checkbox.test.tsx`. +- Existing native callback: `components-checkbox--default` / `toggles native checked state in both directions`, + including assertions in `src/common/desktopFocus.wdio.ts`. +- Generated repository report, only on explicit local opt-in or maintainer-enabled publication: + + ```text + src/components/checkbox/assessment/ASSESSMENT.md + ``` + + This is an output path, not a checked-in file. Generated reports are excluded from npm package contents. + +The knowledge-base guide, machine mapping, and generated report are separate files with different purposes. +The package's `furn.assessments` setting registers the mapping. Shared tooling contains no Checkbox test selection +or requirement-to-assertion policy. + +## Reviewed coverage + +The mapping retains the same 42 reviewed runtime assertions and one native callback from the original pilot. +Additional existing tests remain visible but unmapped until their actual assertions are reviewed. +It groups specific acceptance checks under each CBX requirement, with contract section references, required evidence, +and exact test links. Empty test arrays preserve uncovered checks. Shared check IDs use identical definitions. +The same reviewed mapping is included in local/CI artifacts and the optional atomic publication. + +| Requirement | Current mapped evidence | Remaining limits | +| ---------------------------- | ------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| CBX-001: defaults and API | Selected default JS props/styles | Additional defaults/root props and compiler evidence | +| CBX-002: state and callbacks | Selected JS transitions/ownership; native binary toggle when input is available | Additional callback/precedence assertions; no full requirement claim | +| CBX-003: anatomy and text | Selected JS text, name fallback, indicator and wrapping props | Additional anatomy assertions and rendered geometry | +| CBX-004: accessibility | Selected JS semantics; native checkbox role | Native naming/action dispatch and additional existing assertions | +| CBX-005: styling and tokens | Selected resolved JS tokens and variants | Additional state/style assertions; no visual qualification | +| CBX-006: Win32 focus | No qualifying mapped execution | Focus modality, disabled/detached targets and dual-ring geometry | + +The package uses the React Native macOS Jest mock. Eight assertions simulate Win32 accessibility actions through +`Platform.OS`; they do not prove native UIA dispatch. The Windows/macOS action tables and macOS-only focus case remain +recorded without Win32 requirement credit. A unit-test pass is not native focus, rendered wrapping, or visual qualification. + +Hosted CI disables physical input. The mapped native toggle therefore cannot qualify there merely because navigation +or the enclosing smoke job passed. Missing, skipped, failed or retried execution remains explicit. + +## Mapping review and drift + +Read the SPEC requirements and the exact assertion bodies before changing coverage. Do not infer coverage from test names, +filenames, or green CI. A mapped assertion may prove only part of a requirement, and an existing test may need a mapping +rather than a newly authored duplicate. + +The contract fingerprint covers the listed contract/story files, assertion-bearing helpers, and normalized mapping +content. Only the validated generated SPEC block is removed. The unit fingerprint covers the configured runtime +tests, package/shared Jest and Babel settings, test script/mock profile, capture harness, and normalized mapping. +Line-ending normalization is intentional; changing assertion bodies or mapping scopes invalidates review. +The two reviewed fingerprint fields are excluded from mapping content to avoid self-referential hashes. + +The requirement-first matrix decomposes the SPEC and companions without changing the existing 42-case inventory, +its requirement associations, or the one mapped native callback. It adds explicit uncovered checks rather than +crediting unreviewed tests. The reviewed fingerprints bind this new structure and its precise claims. +Future mapping/fingerprint changes still need review; never refresh a pin just to silence drift. + +For an evidence-free inventory or existing-report replay: + +```powershell +yarn workspace @fluentui-react-native/components assess-components --component checkbox +yarn workspace @fluentui-react-native/components assess-components --component checkbox --unit-evidence "C:\evidence\jest.json" +``` + +The command returns zero when advisory reports are generated. `insufficient-evidence`, null maturity values, and +unmapped cases are expected outcomes, not a readiness approval. Current validation belongs in the PR and CI results, +not historical success/failure snapshots in this guide. diff --git a/packages/agentic/components/references/contracts.md b/packages/agentic/components/references/contracts.md index 10da0530d0..de3b270419 100644 --- a/packages/agentic/components/references/contracts.md +++ b/packages/agentic/components/references/contracts.md @@ -28,6 +28,19 @@ discard the main benefit of YAML comments. Keep this provenance in JSON. `spec/tokens.yaml` remains YAML because it is concise, human-authored token mapping data rather than a generated identity record. +The optional Checkbox assessor snapshot is separate from contract provenance. +It uses a strictly managed `assessment-*` block in the agentic Checkbox +`SPEC.md` front matter; full evidence, digests, and workflow identity remain +in the assessment JSON/report. It never changes `status`, `spec/source.json`, +requirements, or review approval. Only a matched clean execution may populate +its tested commit; local unverified snapshots leave that field null. The +[component assessment guide](assessments/README.md) +defines generation, fingerprinting, and trusted two-file publication. These +assessment fields do not apply to legacy CheckboxV1. +Generated Checkbox assessment reports are source-repository snapshots, not +package contents; they are excluded from the npm inventory. Their directory +is created only when an assessment is explicitly published. + ## Schema version 2 Every source document has the following document-level shape: diff --git a/packages/agentic/components/references/remaining-work.md b/packages/agentic/components/references/remaining-work.md index 5258c6e0ef..25722720a5 100644 --- a/packages/agentic/components/references/remaining-work.md +++ b/packages/agentic/components/references/remaining-work.md @@ -11,6 +11,33 @@ Open work for the agentic component catalog. Completed work is removed rather th is documented in the owning reference, component contract, or the repository focus references (`focus.common`, `focus.windows-win32`, and `focus.macos`). +## Advisory assessment expansion + +- **Outcome:** extend the [assessment workflow](assessments/README.md) using reviewed, package-owned mappings and + existing execution results, without silently qualifying unsupported controls or targets. +- **Current state:** shared discovery, package/Lage commands, per-control reports, and single-invocation unit capture + are implemented; only [Checkbox](assessments/checkbox.md) has a configured mapping. Ordinary PRs assess all configured + controls. Mapping maintenance, readiness policy, and the optional publisher's hosted activation are not automatic. +- **Open gates:** + - inspect actual assertions and review mappings for additional existing Checkbox tests; ingest real compiler/type + results rather than counting a Jest types-suite pass; + - automate the mapping-maintenance phase that reads SPEC states, proposes the requirement/check/test matrix, maps existing + assertions, and distinguishes missing tests from missing mappings; the pilot uses a premade requirement-first checklist, + and owner review must precede authoritative coverage or pin updates; + - replace the premade mapping publisher's exact-source equality rule with trusted regeneration/proposal verification + against the same PR inputs before accepting generated mapping changes; differences from the old checked-in mapping + are expected, but publication must not silently approve new coverage claims or enable report-only reuse of changed inputs; + - select affected controls using implementation, contract, tests, mappings and shared dependency/harness/workflow + changes; ambiguous impact must conservatively include dependents, not silently skip them; + - add another control through its package-owned definition, then review Windows/Fabric and other target adapters using + their existing CI results; Paper, Fabric, mock platforms and native input qualification remain separate; + - agree with owners on Backlog/Experimental/Beta/Production evidence gates and approval policy; until then no stage is assigned; + - validate optional GitHub App publication separately in hosted CI, including failed source checks, branch races, and + atomic publication/report-only reuse when additional controls are configured. +- **Done when:** reviewed mappings and target-specific execution support each added control, missing or unsupported + evidence remains explicit, and owners approve any maturity/publication policy. No input permissions or repository rules + are weakened to make a report green. + ## Desktop focus qualification - **Issues:** [#4274](https://github.com/microsoft/fluentui-react-native/issues/4274); popup-family restoration in diff --git a/packages/agentic/components/src/components/checkbox/assessment.mapping.json b/packages/agentic/components/src/components/checkbox/assessment.mapping.json new file mode 100644 index 0000000000..c0eeac6090 --- /dev/null +++ b/packages/agentic/components/src/components/checkbox/assessment.mapping.json @@ -0,0 +1,961 @@ +{ + "schemaVersion": 2, + "component": "checkbox", + "displayName": "Checkbox", + "componentPath": "src/components/checkbox", + "contract": { + "files": [ + "SPEC.md", + "spec/source.json", + "spec/interaction.md", + "spec/accessibility.md", + "spec/tokens.yaml", + "spec/usage.md", + "checkbox.stories.tsx" + ], + "assertionFiles": ["src/common/desktopFocus.wdio.ts"], + "reviewedDigest": "d6ffc5ba256c3461b801f8446fde1308ac6001c249778f67d9f72dca32ecc3f0", + "mappingVersion": "checkbox-win32-matrix-v2", + "nativeCheck": { + "storyId": "components-checkbox--default", + "testName": "toggles native checked state in both directions", + "requiresPhysicalClick": true + }, + "requirements": { + "CBX-001": { + "checks": [ + { + "id": "default-unchecked", + "description": "Omitting status and defaultStatus exposes checked=false.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "default-enabled", + "description": "Omitting disabled exposes disabled=false.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "default-standard-variant", + "description": "Omitting variant selects the standard indicator radius.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "default-label", + "description": "Omitting label resolves the label text to Label.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "default-visible-label", + "description": "Omitting showLabel leaves the label visible.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "default-secondary-text", + "description": "Omitting secondaryText resolves its value to Description.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "default-hidden-secondary-text", + "description": "Omitting showSecondaryText hides secondary text.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "native-root-props", + "description": "Supported caller Pressable root props reach the root.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "public-prop-types", + "description": "The public types accept the documented status/variant values, callback signature, text/visibility props, and supported Pressable root props.", + "source": "SPEC.md#props-and-defaults", + "evidence": "type", + "tests": [] + }, + { + "id": "optional-status-callback", + "description": "onStatusChange is absent by default; uncontrolled activation remains valid without a supplied callback.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "reject-children", + "description": "The public TypeScript contract rejects caller-supplied children.", + "source": "SPEC.md#props-and-defaults", + "evidence": "type", + "tests": [] + } + ], + "remaining": "Additional defaults/root-prop assertions exist but are not yet mapped; ingest compiler/type results separately." + }, + "CBX-002": { + "checks": [ + { + "id": "uncontrolled-press", + "description": "An uncontrolled unchecked press applies checked and reports checked once while forwarding onPress once.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox toggles uncontrolled status on press and forwards user handlers" }] + }, + { + "id": "controlled-press", + "description": "A controlled unchecked press reports checked without changing the rendered unchecked value.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox reports presses without changing state when the status is externally driven" } + ] + }, + { + "id": "mixed-press", + "description": "An uncontrolled indeterminate press applies checked and reports checked.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox advances indeterminate status to checked on press" }] + }, + { + "id": "checked-press", + "description": "A checked press reports unchecked and applies it only when internally driven.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [] + }, + { + "id": "initial-default-status", + "description": "Without supplied status, each supported defaultStatus value determines the initial rendered status.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "controlled-status-updates", + "description": "Caller changes to supplied status are rendered without the control taking ownership.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [] + }, + { + "id": "status-precedence", + "description": "Supplied status overrides defaultStatus, including when ownership changes.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "press-callback-contract", + "description": "Every controlled/uncontrolled press transition reports the next status once before forwarding onPress.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [] + }, + { + "id": "disabled-press", + "description": "Disabled state is exposed, focusable is false, and a press does not call onPress.", + "source": "spec/accessibility.md#state-and-focus", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox disables interaction and exposes disabled accessibility state" }] + }, + { + "id": "disabled-status-callback", + "description": "A disabled press does not call onStatusChange.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "unit", + "tests": [] + }, + { + "id": "win32-action-unchecked", + "description": "The simulated Win32 Toggle action changes unchecked to checked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles unchecked once without synthesizing a press" } + ] + }, + { + "id": "win32-action-checked", + "description": "The simulated Win32 Toggle action changes checked to unchecked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles checked once without synthesizing a press" } + ] + }, + { + "id": "win32-action-mixed", + "description": "The simulated Win32 Toggle action changes indeterminate to checked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles indeterminate once without synthesizing a press" } + ] + }, + { + "id": "win32-action-labels", + "description": "The simulated Win32 action declaration preserves an already normalized caller action list and labels.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions preserves a normalized caller action list and labels" } + ] + }, + { + "id": "win32-action-deduplication", + "description": "The simulated Win32 action declaration normalizes toggle spellings without losing the first label or custom actions.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions deduplicates legacy and native spellings without losing the caller label or custom actions" + } + ] + }, + { + "id": "win32-controlled-action", + "description": "The simulated controlled Win32 action reports checked once while leaving the supplied mixed state unchanged.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions reports accessibility toggles without changing externally driven status" + } + ] + }, + { + "id": "win32-disabled-action", + "description": "The simulated disabled Win32 action preserves status, suppresses status/press callbacks, and forwards the original action once.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions blocks disabled accessibility toggles while forwarding the caller handler once" + } + ] + }, + { + "id": "win32-other-actions", + "description": "Simulated custom, wrong-case, and activate actions are forwarded once each without changing status or calling status/press callbacks.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions forwards custom, wrong-case, and activate actions without a second activation path" + } + ] + }, + { + "id": "native-binary-toggle", + "description": "Physical clicks change the uncontrolled native checked state from false to true and back to false.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "native", + "tests": [ + { "kind": "native", "storyId": "components-checkbox--default", "testName": "toggles native checked state in both directions" } + ] + }, + { + "id": "whole-row-target", + "description": "The indicator, gap, label, and secondary text activate the same native press target.", + "source": "spec/interaction.md#activation-and-status", + "evidence": "native", + "tests": [] + } + ], + "remaining": "Unlinked checks may have existing tests; native clicks do not prove all ownership/callback combinations." + }, + "CBX-003": { + "checks": [ + { + "id": "anatomy-order", + "description": "The optional focus visual, indicator, label column, label, and secondary text have the documented order.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [] + }, + { + "id": "indicator-box", + "description": "The indicator is always present as a fixed 16-by-16 box.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [] + }, + { + "id": "indicator-glyphs", + "description": "Checked and indeterminate indicators render their respective 12-by-12 glyph nodes.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders checked and indeterminate indicator glyphs" }] + }, + { + "id": "public-slot-surface", + "description": "Root is the only public slot; label, secondary text, indicator, and focus visual remain internal.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "type", + "tests": [] + }, + { + "id": "label-column-presence", + "description": "The label column is present when a text node is rendered and absent when neither text node is rendered.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [] + }, + { + "id": "indicator-symbols", + "description": "Checked draws a checkmark and indeterminate draws a dash.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [] + }, + { + "id": "unchecked-no-glyph", + "description": "The unchecked indicator renders no glyph.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [] + }, + { + "id": "secondary-text-visible", + "description": "Enabling secondary text alongside a visible label renders the supplied supporting text once.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox shows secondary text and exposes it as a description" }] + }, + { + "id": "hidden-label", + "description": "showLabel=false removes visible label text, with either an explicit accessible name or the label fallback.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox keeps the visible label optional while preserving an accessible name" }, + { "kind": "unit", "fullName": "Checkbox falls back to the label text when the visible label is hidden" } + ] + }, + { + "id": "secondary-text-gating", + "description": "Secondary text is suppressed when the visible label is hidden.", + "source": "SPEC.md#requirements", + "evidence": "unit", + "tests": [] + }, + { + "id": "secondary-text-warning", + "description": "A development warning fires when secondary text is requested without a visible label.", + "source": "SPEC.md#requirements", + "evidence": "unit", + "tests": [] + }, + { + "id": "label-shrink", + "description": "Constrained label text resolves flexShrink=1 rather than a truncation policy.", + "source": "SPEC.md#slots-and-anatomy", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox allows constrained label text to wrap" }] + }, + { + "id": "rendered-text-wrapping", + "description": "Label and secondary text actually wrap when the native root width is constrained.", + "source": "spec/usage.md#text", + "evidence": "visual", + "tests": [] + } + ], + "remaining": "Anatomy/order, text gating and warning assertions and native wrapping stories still need reviewed links; style props alone do not prove wrapping geometry." + }, + "CBX-004": { + "checks": [ + { + "id": "js-checkbox-role", + "description": "The JS root declares accessibilityRole=checkbox.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "default-unchecked", + "description": "Omitting status and defaultStatus exposes checked=false.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "default-enabled", + "description": "Omitting disabled exposes disabled=false.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "js-checked-state", + "description": "Supplied checked status exposes checked=true in JS accessibility state.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders checked and indeterminate indicator glyphs" }] + }, + { + "id": "js-mixed-state", + "description": "Supplied indeterminate status exposes checked=mixed in JS accessibility state.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders checked and indeterminate indicator glyphs" }] + }, + { + "id": "label-name", + "description": "Without an explicit accessibilityLabel, the JS accessible name uses the supplied label.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "explicit-visible-name", + "description": "An explicit accessibilityLabel overrides the name fallback even when the visible label is shown.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [] + }, + { + "id": "explicit-hidden-name", + "description": "An explicit accessibilityLabel remains the JS accessible name when visible label text is hidden.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox keeps the visible label optional while preserving an accessible name" }] + }, + { + "id": "fallback-hidden-name", + "description": "The JS accessible name still falls back to label when visible label text is hidden.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox falls back to the label text when the visible label is hidden" }] + }, + { + "id": "secondary-hint", + "description": "Rendered secondary text is delivered as the JS root accessibilityHint.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox shows secondary text and exposes it as a description" }] + }, + { + "id": "secondary-context-not-name", + "description": "Secondary text is supporting context rather than part of the accessible name.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [] + }, + { + "id": "preserve-caller-hint", + "description": "Secondary text is appended after a caller hint using a period and space.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [] + }, + { + "id": "decorative-inner-nodes", + "description": "Inner text and indicator nodes are non-accessible and secondary text is hidden from the accessibility tree.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "unit", + "tests": [] + }, + { + "id": "owned-accessibility-state", + "description": "Resolved role, checked and disabled values override caller values, while other caller accessibility state is preserved.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "accessible-focusable-defaults", + "description": "The enabled root defaults to accessible=true and focusable=true.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [] + }, + { + "id": "disabled-press", + "description": "Disabled state is exposed, focusable is false, and a press does not call onPress.", + "source": "spec/accessibility.md#state-and-focus", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox disables interaction and exposes disabled accessibility state" }] + }, + { + "id": "win32-action-unchecked", + "description": "The simulated Win32 Toggle action changes unchecked to checked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles unchecked once without synthesizing a press" } + ] + }, + { + "id": "win32-action-checked", + "description": "The simulated Win32 Toggle action changes checked to unchecked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles checked once without synthesizing a press" } + ] + }, + { + "id": "win32-action-mixed", + "description": "The simulated Win32 Toggle action changes indeterminate to checked, reports once, forwards the action once afterward, and never synthesizes onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions toggles indeterminate once without synthesizing a press" } + ] + }, + { + "id": "win32-action-labels", + "description": "The simulated Win32 action declaration preserves an already normalized caller action list and labels.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox win32 accessibility actions preserves a normalized caller action list and labels" } + ] + }, + { + "id": "win32-action-deduplication", + "description": "The simulated Win32 action declaration normalizes toggle spellings without losing the first label or custom actions.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions deduplicates legacy and native spellings without losing the caller label or custom actions" + } + ] + }, + { + "id": "win32-controlled-action", + "description": "The simulated controlled Win32 action reports checked once while leaving the supplied mixed state unchanged.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions reports accessibility toggles without changing externally driven status" + } + ] + }, + { + "id": "win32-disabled-action", + "description": "The simulated disabled Win32 action preserves status, suppresses status/press callbacks, and forwards the original action once.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions blocks disabled accessibility toggles while forwarding the caller handler once" + } + ] + }, + { + "id": "win32-other-actions", + "description": "Simulated custom, wrong-case, and activate actions are forwarded once each without changing status or calling status/press callbacks.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "unit", + "tests": [ + { + "kind": "unit", + "fullName": "Checkbox win32 accessibility actions forwards custom, wrong-case, and activate actions without a second activation path" + } + ] + }, + { + "id": "native-checkbox-role", + "description": "The native root exposes the checkbox role.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "native", + "tests": [ + { "kind": "native", "storyId": "components-checkbox--default", "testName": "toggles native checked state in both directions" } + ] + }, + { + "id": "native-three-valued-state", + "description": "Native accessibility exposes unchecked, checked, and mixed values.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "native", + "tests": [] + }, + { + "id": "native-name-and-description", + "description": "Native accessibility exposes the resolved name and supporting context once.", + "source": "spec/accessibility.md#naming-and-description", + "evidence": "native", + "tests": [] + }, + { + "id": "native-toggle-dispatch", + "description": "Actual UIA Toggle invocation changes state and satisfies status/action callback counts without synthesizing onPress.", + "source": "spec/accessibility.md#native-semantics", + "evidence": "native", + "tests": [] + }, + { + "id": "native-disabled-tab", + "description": "A disabled Checkbox is skipped by native keyboard navigation.", + "source": "spec/accessibility.md#state-and-focus", + "evidence": "native", + "tests": [] + } + ], + "remaining": "JS simulation is not native accessibility qualification; native naming/action dispatch and additional existing assertions need reviewed links." + }, + "CBX-005": { + "checks": [ + { + "id": "default-indicator-colors", + "description": "The default unchecked indicator resolves neutral transparent resting fill and neutral heavy resting stroke.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox renders default checkbox accessibility and secondary styling" }] + }, + { + "id": "unchecked-status-colors", + "description": "Unchecked status resolves neutral transparent fill and secondary-emphasis label color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox resolves unchecked status colors" }] + }, + { + "id": "checked-status-colors", + "description": "Checked status resolves brand-heavy fill, primary label color, and on-brand glyph color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox resolves checked status colors" }] + }, + { + "id": "mixed-status-colors", + "description": "Indeterminate status resolves brand-heavy fill, primary label color, and on-brand glyph color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox resolves indeterminate status colors" }] + }, + { + "id": "standard-radius", + "description": "Explicit standard variant selects the base100 indicator radius.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox resolves the standard variant" }] + }, + { + "id": "circular-radius", + "description": "Explicit circular variant selects the circular indicator radius.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox resolves the circular variant" }] + }, + { + "id": "unchecked-hover-colors", + "description": "Hovering an unchecked root selects hover fill, stroke, and secondary-emphasis label colors.", + "source": "spec/interaction.md#state-model", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox resolves hover and pressed feedback for label, indicator, and status colors" } + ] + }, + { + "id": "unchecked-pressed-colors", + "description": "Pressing an already hovered unchecked root selects pressed fill, stroke, and label colors ahead of hover.", + "source": "spec/interaction.md#state-model", + "evidence": "unit", + "tests": [ + { "kind": "unit", "fullName": "Checkbox resolves hover and pressed feedback for label, indicator, and status colors" } + ] + }, + { + "id": "checked-disabled-colors", + "description": "Disabled checked status resolves disabled indicator fill, transparent stroke, and disabled label color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [{ "kind": "unit", "fullName": "Checkbox applies disabled colors to the label and indicator" }] + }, + { + "id": "checked-mixed-stroke", + "description": "Enabled checked and indeterminate status resolve the transparent indicator stroke.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [] + }, + { + "id": "variant-only-radius", + "description": "Standard and circular variants differ only in indicator radius, sharing other indicator values.", + "source": "SPEC.md#props-and-defaults", + "evidence": "unit", + "tests": [] + }, + { + "id": "unchecked-disabled-colors", + "description": "Disabled unchecked status resolves its declared indicator and label colors.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [] + }, + { + "id": "mixed-disabled-colors", + "description": "Disabled indeterminate status resolves its declared indicator and label colors.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [] + }, + { + "id": "disabled-glyph-colors", + "description": "Disabled checked and indeterminate glyphs resolve disabled foreground color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [] + }, + { + "id": "secondary-emphasis-colors", + "description": "Secondary text retains secondary emphasis for every status and uses the declared disabled color.", + "source": "spec/tokens.yaml", + "evidence": "unit", + "tests": [] + }, + { + "id": "secondary-interaction-colors", + "description": "Secondary text uses its declared semantic palette under hovered and pressed state for each status.", + "source": "SPEC.md#requirements", + "evidence": "unit", + "tests": [] + }, + { + "id": "status-interaction-precedence", + "description": "Checked and mixed status retain their semantic palette under hover/press, with disabled overriding both and pressed overriding hovered.", + "source": "spec/interaction.md#state-model", + "evidence": "unit", + "tests": [] + }, + { + "id": "caller-style-last", + "description": "Caller root style overrides token-derived root style.", + "source": "spec/interaction.md#state-model", + "evidence": "unit", + "tests": [] + } + ], + "remaining": "Additional existing state, secondary-text, precedence and caller-style assertions need reviewed links; resolved styles do not prove rendered appearance." + }, + "CBX-006": { + "checks": [ + { + "id": "win32-custom-focus-default", + "description": "Win32 selects the custom focus path by default rather than the system ring.", + "source": "SPEC.md#focus-visuals", + "evidence": "unit", + "tests": [] + }, + { + "id": "keyboard-focus-modality", + "description": "The custom ring is visible only while the root is focused with keyboard modality.", + "source": "SPEC.md#focus-visuals", + "evidence": "native", + "tests": [] + }, + { + "id": "programmatic-focus-modality", + "description": "Programmatic focus uses the last root modality.", + "source": "SPEC.md#focus-visuals", + "evidence": "native", + "tests": [] + }, + { + "id": "disabled-noninteractive-focus", + "description": "Disabled and noninteractive targets never show custom focus feedback.", + "source": "SPEC.md#focus-visuals", + "evidence": "native", + "tests": [] + }, + { + "id": "focus-target-detachment", + "description": "Detaching or disabling a native focus target invalidates pending focus requests.", + "source": "spec/interaction.md#focus-target-lifetime", + "evidence": "native", + "tests": [] + }, + { + "id": "focus-override", + "description": "alwaysVisible selects the custom path and bypasses modality only while focused; useSystemFocusRing remains an explicit composition override.", + "source": "SPEC.md#focus-visuals", + "evidence": "unit", + "tests": [] + }, + { + "id": "focus-ring-lifetime", + "description": "Custom rings remain mounted across focus/blur and do not intercept input.", + "source": "spec/interaction.md#focus-and-motion", + "evidence": "unit", + "tests": [] + }, + { + "id": "focus-target-ref-composition", + "description": "Internal focus-target refs compose with caller refs on the actual interactive root rather than a structural or descendant target.", + "source": "spec/interaction.md#focus-target-lifetime", + "evidence": "unit", + "tests": [] + }, + { + "id": "focus-ref-lifetime", + "description": "Object/callback refs, cleanup, ref replacement, and unmount/remount retain exactly one live focus target.", + "source": "SPEC.md#requirements", + "evidence": "unit", + "tests": [] + }, + { + "id": "self-focus-not-descendant-focus", + "description": "Native self-focus is distinguished from bubbled descendant focus events.", + "source": "spec/interaction.md#focus-target-lifetime", + "evidence": "native", + "tests": [] + }, + { + "id": "focused-modality-update", + "description": "Changing root modality updates an already-focused custom ring without requiring blur/refocus or scene-wide rerenders.", + "source": "SPEC.md#requirements", + "evidence": "native", + "tests": [] + }, + { + "id": "native-custom-exclusivity", + "description": "The actual focus owner uses either the native ring or the custom slot, never competing rings.", + "source": "SPEC.md#requirements", + "evidence": "visual", + "tests": [] + }, + { + "id": "scene-focus-isolation", + "description": "ThemedRoot supplies root modality, with nested theme boundaries sharing the controller and independent scenes retaining separate modality.", + "source": "SPEC.md#focus-visuals", + "evidence": "unit", + "tests": [] + }, + { + "id": "focus-visual-environments", + "description": "Custom ring geometry and shared theme colors remain correct under high contrast, display scaling, clipping, and active/inactive native windows.", + "source": "SPEC.md#requirements", + "evidence": "visual", + "tests": [] + }, + { + "id": "dual-ring-geometry", + "description": "The custom dual ring surrounds the whole row with the shared colors, widths, and resolved component radius.", + "source": "SPEC.md#focus-visuals", + "evidence": "visual", + "tests": [] + } + ], + "remaining": "Win32 focus, target lifetime and visual geometry need qualifying evidence; the recorded macOS mock focus test receives no Win32 credit." + } + } + }, + "unit": { + "testFiles": ["src/components/checkbox/checkbox.test.tsx"], + "inputs": [ + "packages/agentic/components/src/components/checkbox/checkbox.test.tsx", + "packages/agentic/components/jest.config.cjs", + "scripts/configs/jest/jest.config.cjs", + "scripts/configs/jest/babel.config.cjs", + "scripts/src/const.mts", + "scripts/src/tasks/jest.ts", + "scripts/src/ci/assessment/controlAssessmentUnitExecution.ts" + ], + "platform": "macos", + "reviewedDigest": "32892707182052798c9cb0d09b8060966e9b216550ff940aa154ac4b0ccd2089", + "mappingVersion": "checkbox-jest-macos-mock-matrix-v2", + "environment": "React Native macOS Jest mock; explicit platform tables simulate OS values, not native dispatch", + "excludedCases": [ + { + "fullName": "Checkbox windows accessibility actions toggles unchecked once without synthesizing a press", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions toggles checked once without synthesizing a press", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions toggles indeterminate once without synthesizing a press", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions preserves a normalized caller action list and labels", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions deduplicates legacy and native spellings without losing the caller label or custom actions", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions reports accessibility toggles without changing externally driven status", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions blocks disabled accessibility toggles while forwarding the caller handler once", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox windows accessibility actions forwards custom, wrong-case, and activate actions without a second activation path", + "reason": "Windows Platform.OS simulation under the macOS Jest mock; no Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions toggles unchecked once without synthesizing a press", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions toggles checked once without synthesizing a press", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions toggles indeterminate once without synthesizing a press", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions preserves a normalized caller action list and labels", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions deduplicates legacy and native spellings without losing the caller label or custom actions", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions reports accessibility toggles without changing externally driven status", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions blocks disabled accessibility toggles while forwarding the caller handler once", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox macos accessibility actions forwards custom, wrong-case, and activate actions without a second activation path", + "reason": "macOS Platform.OS simulation, not Win32 native accessibility dispatch." + }, + { + "fullName": "Checkbox uses native focus visuals without mounting custom rings", + "reason": "macOS mock focus props, not Win32 custom dual-ring coverage." + } + ] + }, + "limitations": [ + "This is a premade reviewed checklist derived from SPEC requirements and companions, not an automatically generated or exhaustive maturity policy.", + "Empty tests arrays mean no approved mapping for that check, not necessarily that no test exists.", + "Only the existing named native Checkbox callback is mapped; passing it covers binary native clicks and native role, not all state or accessibility behavior.", + "Shared check IDs repeat identical definitions under multiple requirements; all linked tests must qualify before the check passes.", + "The macOS focus case and non-Win32 platform tables remain recorded without Win32 requirement credit." + ] +} diff --git a/scripts/package.json b/scripts/package.json index f4a2cb9479..13a22c9bbf 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -29,7 +29,7 @@ "build": "tsc -b", "format": "node ./src/cli.ts format", "lint": "node ./src/cli.ts lint", - "test": "node --test src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/utils/securityPatches.test.ts src/codegen/file.test.ts src/codegen/valuePool.test.ts" + "test": "node --test \"$@\" src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/utils/securityPatches.test.ts src/codegen/file.test.ts src/codegen/valuePool.test.ts src/ci/assessment/controlAssessment.test.ts src/ci/assessment/controlAssessmentCi.test.ts src/ci/assessment/controlAssessmentUnit.test.ts" }, "dependencies": { "@babel/core": "catalog:", diff --git a/scripts/src/ci/assessment/assessmentConfig.ts b/scripts/src/ci/assessment/assessmentConfig.ts new file mode 100644 index 0000000000..065b6c086c --- /dev/null +++ b/scripts/src/ci/assessment/assessmentConfig.ts @@ -0,0 +1,267 @@ +import { globSync, readFileSync, realpathSync } from 'node:fs'; +import path from 'node:path'; + +import { hex, object, text } from './controlAssessment.ts'; + +export type AssessmentTest = { kind: 'unit'; fullName: string } | { kind: 'native'; storyId: string; testName: string }; +export type AssessmentCheck = { + id: string; + description: string; + source: string; + evidence: 'unit' | 'native' | 'type' | 'visual'; + tests: AssessmentTest[]; +}; +export type AssessmentDefinition = { + schemaVersion: 2; + component: string; + displayName: string; + componentPath: string; + packagePath: string; + packageName: string; + mappingPath: string; + specPath: string; + summaryPath: string; + contract: { + files: string[]; + assertionFiles: string[]; + reviewedDigest: string; + mappingVersion: string; + nativeCheck: { storyId: string; testName: string; requiresPhysicalClick: boolean }; + requirements: Record; + }; + unit: { + testFiles: string[]; + inputs: string[]; + platform: string; + reviewedDigest: string; + mappingVersion: string; + environment: string; + cases: { fullName: string; requirements: string[]; scope: string }[]; + }; + limitations: string[]; +}; + +function relativeFile(value: unknown, label: string): string { + const file = text(value, label); + if ( + file.includes('\\') || + path.posix.isAbsolute(file) || + !file.split('/').every((part) => /^[A-Za-z0-9_.-]+$/.test(part) && part !== '.' && part !== '..') + ) { + throw new Error(`${label} must be a normalized relative repository path.`); + } + return file; +} + +function strings(value: unknown, label: string, files = false): string[] { + if (!Array.isArray(value) || !value.length) throw new Error(`${label} must be a nonempty array.`); + const result = value.map((entry) => (files ? relativeFile(entry, label) : text(entry, label))); + if (new Set(result).size !== result.length) throw new Error(`${label} contains duplicates.`); + return result; +} + +export function readPackageAssessments(root: string, packageRoot: string): AssessmentDefinition[] { + const directory = realpathSync.native(path.resolve(packageRoot)); + const packagePath = path.relative(realpathSync.native(root), directory).replace(/\\/g, '/'); + if (packagePath === '..' || packagePath.startsWith('../') || path.isAbsolute(packagePath)) + throw new Error(`Assessment package must be inside the repository: ${directory} is outside ${root}.`); + const manifest = object(JSON.parse(readFileSync(path.join(directory, 'package.json'), 'utf8')), 'Assessment package'); + if (manifest.furn === undefined) return []; + const profile = object(manifest.furn, 'Package profile'); + if (profile.assessments === undefined) return []; + const mappings = strings(profile.assessments, 'Package assessment mappings', true); + const definitions = mappings.map((file) => { + const absolute = realpathSync.native(path.join(directory, file)); + const relative = path.relative(directory, absolute); + if (relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) + throw new Error('Assessment mapping redirects outside its package.'); + return parseAssessmentDefinition(JSON.parse(readFileSync(absolute, 'utf8')), packagePath, text(manifest.name, 'Package name'), file); + }); + if (new Set(definitions.map(({ component }) => component)).size !== definitions.length) + throw new Error('Duplicate component assessment configuration.'); + for (const definition of definitions) { + if (profile.jestPlatform !== definition.unit.platform) throw new Error('Assessment platform must match the package Jest profile.'); + if (typeof object(manifest.scripts, 'Package scripts').test !== 'string') + throw new Error('Assessment package must declare its test command.'); + } + return definitions; +} + +export function discoverAssessments(root: string, packageRoot?: string): AssessmentDefinition[] { + if (packageRoot) return readPackageAssessments(root, packageRoot); + const manifests = globSync('packages/**/package.json', { + cwd: root, + exclude: ['**/node_modules/**', '**/lib/**', '**/lib-commonjs/**', '**/dist/**'], + }).sort(); + return manifests.flatMap((file) => readPackageAssessments(root, path.dirname(path.join(root, file)))); +} + +export function selectAssessment(root: string, component?: string, packageRoot?: string): AssessmentDefinition { + const definitions = discoverAssessments(root, packageRoot).filter((entry) => component === undefined || entry.component === component); + if (definitions.length !== 1) + throw new Error( + `Expected one configured assessment${component ? ` for ${component}` : ''}; found ${definitions.length}. Select its component and package explicitly.`, + ); + return definitions[0]; +} + +export function parseAssessmentDefinition( + value: unknown, + packagePath: string, + packageName: string, + mappingFile: string, +): AssessmentDefinition { + const data = object(value, 'Assessment mapping'); + if (data.schemaVersion !== 2) throw new Error('Unsupported assessment mapping schema.'); + const component = text(data.component, 'Assessment component'); + if (!/^[a-z][a-z0-9-]*$/.test(component)) throw new Error('Invalid assessment component ID.'); + const componentPath = relativeFile(data.componentPath, 'Component path'); + if (mappingFile !== `${componentPath}/assessment.mapping.json`) throw new Error('Each control mapping must live beside its SPEC.md.'); + const contract = object(data.contract, 'Contract mapping'); + const nativeCheck = object(contract.nativeCheck, 'Native assertion mapping'); + if (typeof nativeCheck.requiresPhysicalClick !== 'boolean') throw new Error('Native input requirement must be explicit.'); + const files = strings(contract.files, 'Contract files', true); + if (!files.includes('SPEC.md') || !files.includes('spec/source.json')) + throw new Error('Assessment requires its SPEC and requirement metadata.'); + const sharedChecks = new Map(); + const cases = new Map }>(); + const requirements = Object.fromEntries( + Object.entries(object(contract.requirements, 'Requirement mappings')).map(([id, value]) => { + if (!/^[A-Z][A-Z0-9]*-\d{3}$/.test(id)) throw new Error('Invalid mapped requirement ID.'); + const entry = object(value, 'Requirement coverage'); + if (!Array.isArray(entry.checks) || !entry.checks.length) throw new Error('Each requirement needs explicit acceptance checks.'); + const checks = entry.checks.map((value): AssessmentCheck => { + const check = object(value, 'Acceptance check'); + const checkId = text(check.id, 'Acceptance check ID'); + if (!/^[a-z][a-z0-9-]*$/.test(checkId)) throw new Error('Invalid acceptance check ID.'); + const source = text(check.source, 'Acceptance check source'); + const [file, anchor, extra] = source.split('#'); + if (!files.includes(file) || (anchor !== undefined && !/^[a-z0-9-]+$/.test(anchor)) || extra !== undefined) + throw new Error('Acceptance checks must reference a declared contract file and optional section anchor.'); + const evidence = check.evidence; + if (evidence !== 'unit' && evidence !== 'native' && evidence !== 'type' && evidence !== 'visual') + throw new Error('Acceptance check must declare its required evidence kind.'); + if (!Array.isArray(check.tests)) throw new Error('Acceptance check tests must be an array, empty when uncovered.'); + const tests = check.tests.map((value): AssessmentTest => { + const test = object(value, 'Acceptance test'); + if (test.kind === 'unit' && evidence === 'unit') return { kind: 'unit', fullName: text(test.fullName, 'Unit assertion name') }; + if ( + test.kind === 'native' && + evidence === 'native' && + test.storyId === nativeCheck.storyId && + test.testName === nativeCheck.testName + ) + return { kind: 'native', storyId: text(test.storyId, 'Mapped story'), testName: text(test.testName, 'Mapped callback') }; + throw new Error('Test must match the required evidence kind and supported native callback; unsupported adapters stay uncovered.'); + }); + if (new Set(tests.map((test) => JSON.stringify(test))).size !== tests.length) throw new Error('Duplicate acceptance test link.'); + const result: AssessmentCheck = { + id: checkId, + description: text(check.description, 'Acceptance check description'), + source, + evidence, + tests, + }; + const serialized = JSON.stringify(result); + if (sharedChecks.has(checkId) && sharedChecks.get(checkId) !== serialized) + throw new Error('Shared acceptance check IDs must have identical definitions under every requirement.'); + sharedChecks.set(checkId, serialized); + for (const test of tests) { + if (test.kind !== 'unit') continue; + const mapped = cases.get(test.fullName) ?? { fullName: test.fullName, requirements: [], scopes: new Set() }; + if (!mapped.requirements.includes(id)) mapped.requirements.push(id); + mapped.scopes.add(result.description); + cases.set(test.fullName, mapped); + } + return result; + }); + if (new Set(checks.map(({ id }) => id)).size !== checks.length) throw new Error('Duplicate acceptance check within a requirement.'); + const observed = checks.filter((check) => check.evidence === 'native' && check.tests.length).map(({ description }) => description); + return [ + id, + { + checks, + ...(observed.length ? { observed: observed.join(' ') } : {}), + remaining: text(entry.remaining, 'Remaining coverage'), + }, + ]; + }), + ); + const unit = object(data.unit, 'Unit mapping'); + if (!Array.isArray(unit.excludedCases)) throw new Error('Unit mapping must declare its excluded-case inventory.'); + const excluded = unit.excludedCases.map((value) => { + const entry = object(value, 'Excluded unit case'); + return { fullName: text(entry.fullName, 'Excluded assertion name'), requirements: [], scope: text(entry.reason, 'Exclusion reason') }; + }); + if (new Set(excluded.map(({ fullName }) => fullName)).size !== excluded.length || excluded.some(({ fullName }) => cases.has(fullName))) + throw new Error('Excluded unit assertions must be unique and cannot receive requirement credit.'); + const testFiles = strings(unit.testFiles, 'Unit runtime files', true); + const inputs = strings(unit.inputs, 'Unit mapping inputs', true); + if (testFiles.some((file) => /\.types?\.test\./.test(file) || !inputs.includes(`${packagePath}/${file}`))) + throw new Error('Each configured runtime suite must be fingerprinted and cannot be a type-only test.'); + const platform = text(unit.platform, 'Unit mock platform'); + if (!['react', 'win32', 'macos', 'ios', 'android', 'windows'].includes(platform)) + throw new Error('Unsupported assessment Jest platform.'); + return { + schemaVersion: 2, + component, + displayName: text(data.displayName, 'Component display name'), + componentPath, + packagePath, + packageName, + mappingPath: `${packagePath}/${mappingFile}`, + specPath: `${packagePath}/${componentPath}/SPEC.md`, + summaryPath: `${packagePath}/${componentPath}/assessment/ASSESSMENT.md`, + contract: { + files, + assertionFiles: strings(contract.assertionFiles, 'Native assertion files', true), + reviewedDigest: hex(contract.reviewedDigest, 64, 'Reviewed contract digest'), + mappingVersion: text(contract.mappingVersion, 'Native mapping version'), + nativeCheck: { + storyId: text(nativeCheck.storyId, 'Mapped story'), + testName: text(nativeCheck.testName, 'Mapped callback'), + requiresPhysicalClick: nativeCheck.requiresPhysicalClick, + }, + requirements, + }, + unit: { + testFiles, + inputs, + platform, + reviewedDigest: hex(unit.reviewedDigest, 64, 'Reviewed unit digest'), + mappingVersion: text(unit.mappingVersion, 'Unit mapping version'), + environment: text(unit.environment, 'Unit environment'), + cases: [...cases.values()].map(({ scopes, ...entry }) => ({ ...entry, scope: [...scopes].join(' ') })).concat(excluded), + }, + limitations: strings(data.limitations, 'Component assessment limitations'), + }; +} + +export function mappingContent(definition: AssessmentDefinition): string { + const { reviewedDigest: _contractDigest, ...contract } = definition.contract; + const { reviewedDigest: _unitDigest, ...unit } = definition.unit; + return JSON.stringify({ ...definition, contract, unit }); +} + +export function readMappingContent(read: (file: string) => string, definition: AssessmentDefinition): string { + const mappingFile = definition.mappingPath.slice(definition.packagePath.length + 1); + return mappingContent( + parseAssessmentDefinition(JSON.parse(read(definition.mappingPath)), definition.packagePath, definition.packageName, mappingFile), + ); +} + +export function assessmentDirectory(root: string, definition: AssessmentDefinition): string { + return path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment-ci', definition.packagePath, definition.component); +} + +export function unitCaptureDirectory(root: string, definition: AssessmentDefinition): string { + return path.join(root, definition.packagePath, definition.component); +} + +export function unconfiguredControls(packageRoot: string, definitions: AssessmentDefinition[]): string[] { + return globSync('src/components/*/SPEC.md', { cwd: packageRoot }) + .map((file) => path.dirname(file).replace(/\\/g, '/')) + .filter((directory) => !definitions.some(({ componentPath }) => componentPath === directory)) + .map((directory) => path.posix.basename(directory)) + .sort(); +} diff --git a/scripts/src/ci/assessment/assessmentMarkdown.ts b/scripts/src/ci/assessment/assessmentMarkdown.ts new file mode 100644 index 0000000000..2434775ef1 --- /dev/null +++ b/scripts/src/ci/assessment/assessmentMarkdown.ts @@ -0,0 +1,23 @@ +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const toolingRoot = fileURLToPath(new URL('../../../../', import.meta.url)); +const require = createRequire(import.meta.url); + +export function formatAssessmentDocument(content: string, file = 'ASSESSMENT.md'): string { + const manifest = require.resolve('oxfmt/package.json', { + paths: [process.env.FURN_ASSESSMENT_TOOLS ?? toolingRoot], + }); + const command = path.join(path.dirname(manifest), 'bin', 'oxfmt'); + const result = spawnSync( + process.execPath, + [command, `--stdin-filepath=${file}`, `--config=${path.join(toolingRoot, '.oxfmtrc.json')}`, '--disable-nested-config', '--threads=1'], + { cwd: toolingRoot, input: content, encoding: 'utf8', maxBuffer: 8_000_000 }, + ); + if (result.error) throw result.error; + if (result.status !== 0 || !result.stdout.trim()) + throw new Error(`Assessment Markdown formatting failed: ${result.stderr || `exit ${result.status}`}`); + return result.stdout; +} diff --git a/scripts/src/ci/assessment/controlAssessment.test.ts b/scripts/src/ci/assessment/controlAssessment.test.ts new file mode 100644 index 0000000000..b07a0c5ef0 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessment.test.ts @@ -0,0 +1,828 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, linkSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { + assessControl, + formatAssessmentMarkdown, + parseEvidenceContext, + parseNativeReport, + readControlContract, + readNativeEvidence, + reportDigest, + win32StorybookTargetId, +} from './controlAssessment.ts'; +import type { AssessmentOptions, EvidenceContext } from './controlAssessment.ts'; +import { + assessmentDefinition, + assessmentMappingPath, + assessmentPackagePath, + assessmentSpecPath, + checkboxJestFixture, + checkboxNativeAssertionPath, + desktopEvidenceFixture, + readControlContractFrom, +} from './controlAssessmentFixtures.ts'; +import { specAssessmentFor, splitSpecAssessment, updateSpecAssessment } from './controlAssessmentSpec.ts'; + +const root = fileURLToPath(new URL('../../../../', import.meta.url)); +const contract = readControlContract(root); +const sourceCommit = 'a'.repeat(40); +const platformManifestDigest = 'b'.repeat(64); +const now = new Date('2026-10-21T21:00:00.000Z'); +const nativeCase = { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'passed', + durationMs: 3687, +}; +const direct = { endpoint: 'win32', platformManifestDigest, status: 'passed', tests: [nativeCase] }; +const aggregate = { + schemaVersion: 1, + endpoint: 'win32', + platformName: 'windows', + targetId: 'agenticstorybook-win32', + runId: 'synthetic-run', + startedAt: '2026-10-21T19:59:00.000Z', + finishedAt: '2026-10-21T20:00:00.000Z', + manifest: { platform: platformManifestDigest, portable: 'c'.repeat(64) }, + status: 'passed', + tests: [{ storyId: 'ignored-portable-test', status: 'passed' }], + wdio: [{ ...nativeCase, status: 'skipped', skipReason: 'This test requires physical pointer input.' }], +}; +const raw = JSON.stringify(direct); +const context: EvidenceContext = { + schemaVersion: 1, + reportSha256: reportDigest(raw), + sourceCommit, + buildId: 'synthetic-build', + contractDigest: contract.digest, + platformManifestDigest, + endpoint: 'win32', + renderer: 'paper', + targetId: 'agenticstorybook-win32', + finishedAt: '2026-10-21T20:00:00.000Z', + physicalClick: true, +}; + +function options(overrides: Partial = {}): AssessmentOptions { + return { + contract, + sourceCommit, + workingTreeDirty: false, + now, + report: parseNativeReport(direct), + reportSha256: reportDigest(raw), + context, + expectedBuildId: context.buildId, + maxAgeHours: 24, + ...overrides, + }; +} + +const fixtures: string[] = []; +function fixture(): string { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-assessment-')); + fixtures.push(directory); + return directory; +} +afterEach(() => { + for (const directory of fixtures.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('Checkbox advisory assessment', () => { + it('loads six reviewed requirements without converting implemented/reviewed into a maturity stage', () => { + assert.equal(contract.mappingCurrent, true); + assert.deepEqual( + contract.requirements.map(({ id }) => id), + ['CBX-001', 'CBX-002', 'CBX-003', 'CBX-004', 'CBX-005', 'CBX-006'], + ); + const report = assessControl(options()); + assert.equal(report.contract.lifecycle, 'implemented'); + assert.equal(report.contract.conformance, 'reviewed'); + assert.equal(report.declaredStatus, null); + assert.equal(report.approvedStatus, null); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.policyStatus, 'pending'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.deepEqual( + report.requirements.map(({ outcome }) => outcome), + ['insufficient-evidence', 'partial', 'insufficient-evidence', 'partial', 'insufficient-evidence', 'insufficient-evidence'], + ); + assert.equal(report.evidence.caseCounts.passed, 1); + assert.equal(report.requirements.filter(({ check }) => check).length, 2); + assert.ok(report.requirements.every(({ remainingCoverage }) => remainingCoverage.length > 0)); + }); + + describe('desktop-test canonical evidence', () => { + const smoke = JSON.stringify(aggregate); + it('computes the same Win32/Paper target identity as the actual upstream configuration resolver', () => { + const directory = fixture(); + const appRoot = path.join(directory, 'apps', 'storybook'); + mkdirSync(appRoot, { recursive: true }); + const configPath = path.join(appRoot, 'desktop.config.mts'); + writeFileSync(configPath, 'export default { schemaVersion: 1, targets: { win32: { host: { kind: "rex", renderer: "paper" } } } };'); + const resolver = pathToFileURL(path.join(root, 'packages', 'agentic', 'desktop-test', 'src', 'config.ts')).href; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '-e', + 'const { loadDesktopConfig } = await import(process.argv[1]); const { targets } = await loadDesktopConfig(process.argv[2]); console.log(targets.win32.id);', + resolver, + configPath, + ], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(win32StorybookTargetId(directory), result.stdout.trim()); + }); + + it('binds generated target IDs without treating an arbitrary Win32 instance as the expected host', () => { + const targetId = `win32-${'1'.repeat(16)}`; + const { envelope } = desktopEvidenceFixture(JSON.stringify({ ...aggregate, targetId })); + const report = parseNativeReport(JSON.parse(envelope)); + const bound = parseEvidenceContext({ ...context, reportSha256: reportDigest(envelope), targetId }); + const input = options({ report, reportSha256: reportDigest(envelope), context: bound, expectedTargetId: targetId }); + assert.equal(assessControl(input).evidence.status, 'matched'); + assert.equal(assessControl({ ...input, expectedTargetId: undefined }).evidence.status, 'unverified'); + assert.equal(assessControl({ ...input, expectedTargetId: `win32-${'2'.repeat(16)}` }).evidence.status, 'mismatched'); + assert.equal(assessControl({ ...input, context: { ...bound, targetId: 'agenticstorybook-win32' } }).evidence.status, 'mismatched'); + assert.equal(assessControl({ ...input, context: undefined, expectedTargetId: undefined }).evidence.status, 'unverified'); + }); + + it('combines canonical supervisor and smoke evidence without counting navigation as requirement coverage', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke).envelope)); + assert.equal(result.format, 'desktop-test'); + assert.equal(result.harness?.status, 'passed'); + assert.equal(result.cases.length, 1); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.harness?.retries, 0); + }); + + it('loads a canonical run with its declared companion, preserving both sets of bytes', () => { + const directory = fixture(); + const { canonical } = desktopEvidenceFixture(smoke); + const evidence = path.join(directory, 'evidence', 'win32', 'static'); + mkdirSync(evidence, { recursive: true }); + writeFileSync(path.join(directory, 'run.json'), canonical); + writeFileSync(path.join(evidence, 'run.json'), smoke); + const content = readNativeEvidence(path.join(directory, 'run.json')); + assert.equal(JSON.parse(content).desktopRun, canonical); + assert.equal(JSON.parse(content).storyReport, smoke); + assert.equal(parseNativeReport(JSON.parse(content)).format, 'desktop-test'); + }); + + it('rejects a missing companion instead of silently accepting supervisor success', () => { + const directory = fixture(); + writeFileSync(path.join(directory, 'run.json'), desktopEvidenceFixture(smoke).canonical); + assert.throws(() => readNativeEvidence(path.join(directory, 'run.json')), /ENOENT/); + }); + + it('retains cleanup failure even when the Storybook evidence passed', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + status: 'failed', + failures: [{ failureId: 'cleanup', kind: 'cleanup', error: { name: 'Error', message: 'session cleanup failed' } }], + }); + const result = parseNativeReport(JSON.parse(envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.failureCount, 1); + assert.match(result.error ?? '', /supervisor status: failed/); + }); + + it('preserves unfinished supervisor execution rather than qualifying its recorded cases', () => { + const result = parseNativeReport(JSON.parse(desktopEvidenceFixture(smoke, { status: 'incomplete' }).envelope)); + assert.equal(result.status, 'failed'); + assert.equal(result.harness?.status, 'incomplete'); + }); + + it('does not accept retry/flaky passes as a clean qualifying execution', () => { + const passedSmoke = JSON.stringify({ ...aggregate, wdio: [nativeCase] }); + const { run } = desktopEvidenceFixture(passedSmoke); + const retried = run.tests[1]; + retried.attempts.unshift({ ...retried.attempts[0] }); + retried.attempts[1].attempt = 1; + run.counts.retries = 1; + const result = parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: passedSmoke, + }); + assert.equal(result.harness?.retries, 1); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /Retried or flaky/); + }); + + for (const [name, mutate] of [ + [ + 'counts', + (run: Record) => { + run.counts = {}; + }, + ], + [ + 'duplicated artifact', + (run: Record) => { + run.artifacts = [{ path: 'evidence/win32/static/run.json' }, { path: 'evidence\\win32\\static\\run.json' }]; + }, + ], + [ + 'wrong run kind', + (run: Record) => { + run.kind = 'lifecycle'; + }, + ], + [ + 'outside timestamp', + (run: Record) => { + run.startedAt = '2026-10-22T00:00:00.000Z'; + }, + ], + [ + 'invalid schema', + (run: Record) => { + run.schemaVersion = 2; + }, + ], + ] as const) { + it(`rejects inconsistent ${name}`, () => { + const run: Record = JSON.parse(desktopEvidenceFixture(smoke).canonical); + mutate(run); + assert.throws(() => + parseNativeReport({ schemaVersion: 1, format: 'desktop-test', desktopRun: JSON.stringify(run), storyReport: smoke }), + ); + }); + } + + it('rejects a callback status that contradicts the canonical result', () => { + const { canonical } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: JSON.stringify({ ...aggregate, wdio: [nativeCase] }), + }), + /does not match/, + ); + }); + + it('rejects a nested evidence envelope instead of recursively trusting it as a companion', () => { + const { canonical, envelope } = desktopEvidenceFixture(smoke); + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: canonical, + storyReport: envelope, + }), + /companion/, + ); + }); + + it('rejects successful supervisor status when selected work is unfinished', () => { + const { envelope } = desktopEvidenceFixture(smoke, { + work: [{ workId: 'unfinished', label: 'selected test cohort', status: 'not-run' }], + }); + assert.throws(() => parseNativeReport(JSON.parse(envelope)), /incomplete execution/); + }); + + it('rejects a native identity that is inconsistent with its canonical test ID', () => { + const { run } = desktopEvidenceFixture(smoke); + run.tests[1].testId = `csf-${'0'.repeat(64)}`; + assert.throws( + () => + parseNativeReport({ + schemaVersion: 1, + format: 'desktop-test', + desktopRun: JSON.stringify(run), + storyReport: smoke, + }), + /authored callback identity/, + ); + }); + + it('accepts the new direct optional-cohort skipped status without calling it a pass', () => { + const result = parseNativeReport({ + ...direct, + status: 'skipped', + tests: [{ ...nativeCase, status: 'skipped', skipReason: 'physical input disabled' }], + }); + assert.equal(result.cases[0].status, 'skipped'); + assert.equal(result.status, 'failed'); + assert.match(result.error ?? '', /skipped/); + }); + }); + + it('records a raw historical pass without accepting it as current qualification', () => { + const report = assessControl(options({ context: undefined, expectedBuildId: undefined, maxAgeHours: undefined })); + assert.equal(report.evidence.status, 'unverified'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('uses aggregate wdio results, not enclosing success or portable tests', () => { + const report = assessControl(options({ report: parseNativeReport(aggregate), context: undefined })); + assert.equal(report.evidence.runStatus, 'passed'); + assert.equal(report.evidence.caseCounts.skipped, 1); + assert.equal(report.evidence.caseCounts.passed, 0); + assert.equal(report.requirements[1].check?.recordedOutcome, 'skipped'); + assert.match(report.requirements[1].check?.reason ?? '', /physical pointer/); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + }); + + it('keeps a matched skip as a skip and never counts it as a pass', () => { + const report = assessControl( + options({ + report: parseNativeReport(aggregate), + context: { ...context, physicalClick: false }, + }), + ); + assert.equal(report.evidence.status, 'matched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('blocks mapped evidence on a case failure without attributing the precise broken requirement', () => { + const report = assessControl( + options({ + report: parseNativeReport({ ...direct, status: 'failed', tests: [{ ...nativeCase, status: 'failed', error: 'assertion failed' }] }), + }), + ); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[1].check?.reason, 'assertion failed'); + }); + + it('separates runner errors from product assertion failures and does not accept partial-run passes', () => { + const report = assessControl(options({ report: parseNativeReport({ ...direct, status: 'failed', error: 'driver disconnected' }) })); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.equal(report.evidence.runError, 'driver disconnected'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'passed'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'unverified'); + }); + + for (const report of [ + undefined, + parseNativeReport({ ...direct, tests: [] }), + parseNativeReport({ ...direct, tests: [{ ...nativeCase, storyId: 'components-button--default' }] }), + ]) { + it(`reports missing coverage for ${report === undefined ? 'absent report' : report.cases.length ? 'unrelated cases' : 'empty cases'}`, () => { + const result = assessControl(options({ report, context: undefined })); + assert.equal(result.requirements[1].check?.recordedOutcome, 'not-run'); + assert.equal(result.requirements[1].outcome, 'insufficient-evidence'); + }); + } + + it('detects endpoint and native-host mismatches even without provenance context', () => { + const report = assessControl(options({ report: parseNativeReport({ ...direct, endpoint: 'windows' }), context: undefined })); + assert.equal(report.assessmentHealth, 'blocked'); + assert.equal(report.evidence.status, 'mismatched'); + const host = assessControl(options({ report: parseNativeReport({ ...aggregate, targetId: 'different-host' }), context: undefined })); + assert.equal(host.evidence.status, 'mismatched'); + }); + + for (const [field, value] of Object.entries({ + reportSha256: '1'.repeat(64), + sourceCommit: '2'.repeat(40), + contractDigest: '3'.repeat(64), + platformManifestDigest: '4'.repeat(64), + buildId: 'another-build', + finishedAt: '2026-10-22T00:00:00.000Z', + physicalClick: false, + })) { + it(`rejects mismatched ${field} provenance`, () => { + const report = assessControl(options({ context: parseEvidenceContext({ ...context, [field]: value }) })); + assert.equal(report.evidence.status, 'mismatched'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'mismatched'); + assert.equal(report.evidenceSupportedStatus, null); + }); + } + + it('does not accept a context timestamp that contradicts the aggregate', () => { + const report = assessControl( + options({ report: parseNativeReport(aggregate), context: { ...context, finishedAt: '2026-10-21T20:01:00.000Z' } }), + ); + assert.equal(report.evidence.status, 'mismatched'); + }); + + it('requires explicit freshness/build policy and a commit-identifiable working tree', () => { + for (const missing of [{ maxAgeHours: undefined }, { expectedBuildId: undefined }, { workingTreeDirty: true }]) { + assert.equal(assessControl(options(missing)).evidence.status, 'unverified'); + } + assert.equal(assessControl(options({ maxAgeHours: 1 })).evidence.status, 'matched'); + const stale = assessControl(options({ maxAgeHours: 0.5 })); + assert.equal(stale.assessmentHealth, 'stale'); + assert.equal(stale.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(stale.evidenceSupportedStatus, null); + }); + + it('invalidates changed reviewed inputs and never promotes an unmapped requirement', () => { + const report = assessControl( + options({ + contract: { + ...contract, + mappingCurrent: false, + requirements: [...contract.requirements, { id: 'CBX-007', summary: 'New contract requirement', declaredEvidence: [] }], + }, + }), + ); + assert.equal(report.assessmentHealth, 'stale'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'stale'); + assert.equal(report.requirements[6].check, null); + assert.equal(report.requirements[6].outcome, 'insufficient-evidence'); + }); + + it('fingerprints normalized contract/story content and detects a changed source file', () => { + const directory = fixture(); + const component = 'packages/agentic/components/src/components/checkbox'; + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const destination = path.join(directory, component, file); + mkdirSync(path.dirname(destination), { recursive: true }); + copyFileSync(path.join(root, component, file), destination); + writeFileSync(destination, readFileSync(destination, 'utf8').replace(/\r?\n/g, '\r\n')); + } + const helper = path.join(directory, checkboxNativeAssertionPath); + mkdirSync(path.dirname(helper), { recursive: true }); + writeFileSync(helper, readFileSync(path.join(root, checkboxNativeAssertionPath), 'utf8').replace(/\r?\n/g, '\r\n')); + copyFileSync(path.join(root, assessmentMappingPath), path.join(directory, assessmentMappingPath)); + assert.equal(readControlContract(directory, assessmentDefinition).mappingCurrent, true); + writeFileSync(path.join(directory, component, 'checkbox.stories.tsx'), '// changed test\n'); + assert.equal(readControlContract(directory, assessmentDefinition).mappingCurrent, false); + }); + + it('invalidates native mappings when only the assertion-bearing helper changes', () => { + const changed = readControlContractFrom((file) => { + const content = readFileSync(path.join(root, file), 'utf8'); + return file === checkboxNativeAssertionPath ? `${content}\n// changed native assertion\n` : content; + }); + assert.notEqual(changed.digest, contract.digest); + assert.equal(changed.mappingCurrent, false); + }); + + it('produces deterministic decisions and escapes untrusted Markdown', () => { + assert.deepEqual(assessControl(options()), assessControl(options())); + for (const tag of ['script', 'SCRIPT', 'ScRiPt']) { + const report = assessControl( + options({ + report: parseNativeReport({ + ...direct, + tests: [{ ...nativeCase, status: 'skipped', skipReason: `<${tag}>|[link](bad)\nsecond line` }], + }), + context: undefined, + }), + ); + const output = formatAssessmentMarkdown(report); + assert.match(output, /not established/); + assert.ok(output.includes(`\\<${tag}\\>\\|\\[link\\]`)); + assert.equal(output.includes(`<${tag}>`), false); + } + }); +}); + +describe('native evidence validation', () => { + const malformed: [string, unknown][] = [ + ['non-object input', null], + ['missing case array', { ...direct, tests: undefined }], + ['unknown case status', { ...direct, tests: [{ ...nativeCase, status: 'pending' }] }], + ['unknown root status', { ...direct, status: 'success' }], + ['bad platform digest', { ...direct, platformManifestDigest: 'bad' }], + ['negative duration', { ...direct, tests: [{ ...nativeCase, durationMs: -1 }] }], + ['non-finite duration', { ...direct, tests: [{ ...nativeCase, durationMs: Infinity }] }], + ['missing duration', { ...direct, tests: [{ ...nativeCase, durationMs: undefined }] }], + ['duplicate native identity', { ...direct, tests: [nativeCase, nativeCase] }], + ['missing skip reason', { ...direct, tests: [{ ...nativeCase, status: 'skipped' }] }], + ['invalid error', { ...direct, status: 'failed', error: {} }], + ['invalid case error', { ...direct, tests: [{ ...nativeCase, error: {} }] }], + ['passed report with failed case', { ...direct, tests: [{ ...nativeCase, status: 'failed' }] }], + ['passed report with runner error', { ...direct, error: 'driver failed' }], + ['passed case with error', { ...direct, tests: [{ ...nativeCase, error: 'failed' }] }], + ['passed case with evidence error', { ...direct, tests: [{ ...nativeCase, evidenceErrors: ['capture failed'] }] }], + ['invalid evidence errors', { ...direct, tests: [{ ...nativeCase, evidenceErrors: 'capture failed' }] }], + ['unknown smoke schema', { ...aggregate, schemaVersion: 2 }], + ['missing smoke wdio', { ...aggregate, wdio: undefined }], + ['bad smoke digest', { ...aggregate, manifest: { platform: 'bad', portable: 'c'.repeat(64) } }], + ['contradictory smoke digest', { ...aggregate, platformManifestDigest: 'd'.repeat(64) }], + ['smoke time reversal', { ...aggregate, startedAt: '2026-10-22T00:00:00.000Z' }], + ['impossible date', { ...aggregate, finishedAt: '2026-02-30T20:00:00.000Z' }], + ]; + for (const [label, value] of malformed) it(`rejects ${label}`, () => assert.throws(() => parseNativeReport(value))); + + for (const [field, value] of Object.entries({ + schemaVersion: 2, + reportSha256: 'bad', + sourceCommit: 'HEAD', + buildId: '', + contractDigest: 'bad', + platformManifestDigest: 'bad', + endpoint: 'windows', + renderer: 'fabric', + targetId: 'another-host', + finishedAt: '2026-02-30T20:00:00.000Z', + physicalClick: 'true', + })) { + it(`rejects invalid context ${field}`, () => assert.throws(() => parseEvidenceContext({ ...context, [field]: value }))); + } + it('round-trips a valid execution context', () => assert.deepEqual(parseEvidenceContext(context), context)); + it('rejects context without report, invalid target identity, and invalid age options', () => { + assert.throws(() => assessControl(options({ report: undefined }))); + assert.throws(() => assessControl(options({ sourceCommit: 'HEAD' }))); + for (const maxAgeHours of [0, -1, NaN, Infinity]) assert.throws(() => assessControl(options({ maxAgeHours }))); + }); +}); + +describe('assess-control command', () => { + function run(args: string[]) { + return spawnSync(process.execPath, [path.join(root, 'scripts', 'src', 'cli.ts'), 'assess-control', ...args], { + cwd: root, + encoding: 'utf8', + }); + } + + it('lists owned mappings and explicitly unconfigured controls without producing readiness claims', () => { + const result = run(['--package', path.join(root, assessmentDefinition.packagePath), '--list']); + assert.equal(result.status, 0, result.stderr || result.stdout); + const inventory = JSON.parse(result.stdout); + assert.equal(inventory.configured.length, 1); + assert.equal(inventory.configured[0].component, 'checkbox'); + assert.ok( + inventory.unconfigured.some( + (entry: { component: string; status: string }) => entry.component === 'button' && entry.status === 'not-configured', + ), + ); + assert.equal(Object.hasOwn(inventory, 'approvedStatus'), false); + }); + + it('consumes runtime unit JSON without inventing provenance, and protects unit inputs from overwrite', () => { + const directory = fixture(); + const input = path.join(directory, 'jest.json'); + const raw = JSON.stringify(checkboxJestFixture()); + writeFileSync(input, raw); + const result = run(['--component', 'checkbox', '--unit-evidence', input, '--output', directory]); + assert.equal(result.status, 0, result.stderr || result.stdout); + const report = JSON.parse(readFileSync(path.join(directory, 'assessment.json'), 'utf8')); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.status, 'unverified'); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.inputs.unitReport, input); + const other = fixture(); + linkSync(input, path.join(other, 'assessment.json')); + assert.notEqual(run(['--component', 'checkbox', '--unit-evidence', input, '--output', other]).status, 0); + assert.equal(readFileSync(input, 'utf8'), raw); + assert.notEqual(run(['--component', 'checkbox', '--run-unit-tests', '--unit-evidence', input]).status, 0); + assert.notEqual(run(['--component', 'checkbox', '--unit-context', input]).status, 0); + assert.notEqual(run(['--component', 'checkbox', '--unit-build-id', 'missing-report']).status, 0); + }); + + it('persists readable/JSON reports and leaves input bytes unchanged', () => { + const directory = fixture(); + const input = path.join(directory, 'run.json'); + writeFileSync(input, JSON.stringify(aggregate)); + const before = readFileSync(input, 'utf8'); + const output = path.join(directory, 'reports'); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', output]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.match(result.stdout, /insufficient-evidence; readiness not established/); + const report = JSON.parse(readFileSync(path.join(output, 'assessment.json'), 'utf8')); + assert.equal(report.requirements[1].check.recordedOutcome, 'skipped'); + assert.equal(report.evidenceSupportedStatus, null); + assert.match(readFileSync(path.join(output, 'assessment.md'), 'utf8'), /physical pointer input/); + assert.equal(readFileSync(input, 'utf8'), before); + }); + + it('produces an explicit evidence inventory when no report is supplied', () => { + const directory = fixture(); + const result = run(['--component', 'checkbox', '--output', directory]); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(JSON.parse(readFileSync(path.join(directory, 'assessment.json'), 'utf8')).evidence.status, 'missing'); + }); + + it('fails for malformed/missing input, unsupported controls, and attempted input overwrite', () => { + const directory = fixture(); + const input = path.join(directory, 'assessment.json'); + for (const content of ['{', raw]) { + writeFileSync(input, content); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(input, 'utf8'), content); + } + assert.notEqual(run(['--component', 'checkbox', '--evidence', path.join(directory, 'missing.json'), '--output', directory]).status, 0); + assert.notEqual(run(['--component', 'button', '--output', directory]).status, 0); + }); + + it('does not overwrite evidence through a different hard-linked filename', () => { + const directory = fixture(); + const input = path.join(directory, 'run.json'); + writeFileSync(input, raw); + linkSync(input, path.join(directory, 'assessment.json')); + const result = run(['--component', 'checkbox', '--evidence', input, '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(input, 'utf8'), raw); + }); + + it('rejects JSON and Markdown outputs that alias one another', () => { + const directory = fixture(); + const output = path.join(directory, 'assessment.json'); + writeFileSync(output, 'existing report'); + linkSync(output, path.join(directory, 'assessment.md')); + const result = run(['--component', 'checkbox', '--output', directory]); + assert.notEqual(result.status, 0); + assert.equal(readFileSync(output, 'utf8'), 'existing report'); + }); + + it('updates SPEC only on explicit opt-in, preserves its contract and records honest local identity', () => { + const directory = fixture(); + const checkout = path.join(directory, 'repo'); + mkdirSync(checkout); + const invokeGit = (...args: string[]) => { + const result = spawnSync('git', args, { cwd: checkout, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout.trim(); + }; + invokeGit('init', '-q'); + invokeGit('config', 'core.autocrlf', 'false'); + const component = path.dirname(assessmentSpecPath); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const target = path.join(checkout, component, file); + mkdirSync(path.dirname(target), { recursive: true }); + copyFileSync(path.join(root, component, file), target); + } + const helper = path.join(checkout, checkboxNativeAssertionPath); + mkdirSync(path.dirname(helper), { recursive: true }); + copyFileSync(path.join(root, checkboxNativeAssertionPath), helper); + copyFileSync(path.join(root, assessmentMappingPath), path.join(checkout, assessmentMappingPath)); + writeFileSync( + path.join(checkout, assessmentPackagePath), + JSON.stringify({ ...JSON.parse(readFileSync(path.join(root, assessmentPackagePath), 'utf8')), version: '1.2.3' }), + ); + invokeGit('add', '.'); + invokeGit( + '-c', + 'user.name=Fixture', + '-c', + 'user.email=fixture@example.invalid', + '-c', + 'commit.gpgsign=false', + 'commit', + '-qm', + 'Fixture', + ); + const revision = invokeGit('rev-parse', 'HEAD'); + const specPath = path.join(checkout, assessmentSpecPath); + const original = readFileSync(specPath, 'utf8'); + const input = path.join(directory, 'run.json'); + writeFileSync(input, raw); + const localRun = (extra: string[] = []) => + spawnSync( + process.execPath, + [ + path.join(root, 'scripts', 'src', 'cli.ts'), + 'assess-control', + '--component', + 'checkbox', + '--evidence', + input, + '--output', + path.join(directory, 'output'), + ...extra, + ], + { cwd: checkout, encoding: 'utf8' }, + ); + const ordinary = localRun(); + assert.equal(ordinary.status, 0, ordinary.stderr || ordinary.stdout); + assert.equal(readFileSync(specPath, 'utf8'), original); + assert.equal(invokeGit('status', '--porcelain'), ''); + writeFileSync(path.join(checkout, 'uncommitted.txt'), 'Local source is dirty.\n'); + const update = localRun(['--update-spec']); + assert.equal(update.status, 0, update.stderr || update.stdout); + const snapshot = splitSpecAssessment(readFileSync(specPath, 'utf8')); + assert.equal(snapshot.contract, splitSpecAssessment(original).contract); + assert.equal(snapshot.assessment?.packageVersion, '1.2.3'); + assert.equal(snapshot.assessment?.sourceCommit, revision); + assert.equal(snapshot.assessment?.testedCommit, null); + assert.equal(snapshot.assessment?.dirty, true); + assert.equal(snapshot.assessment?.provenance, 'unverified'); + assert.equal(snapshot.assessment?.outcome, 'insufficient-evidence'); + const summary = path.join(path.dirname(specPath), 'assessment', 'ASSESSMENT.md'); + assert.match(readFileSync(summary, 'utf8'), /Not a bound CI publication/); + assert.equal(readFileSync(input, 'utf8'), raw); + linkSync(summary, path.join(directory, 'summary-alias.md')); + const linked = localRun(['--update-spec']); + assert.notEqual(linked.status, 0); + assert.match(linked.stderr || linked.stdout, /regular, unlinked/); + assert.equal(splitSpecAssessment(readFileSync(specPath, 'utf8')).contract, splitSpecAssessment(original).contract); + }); +}); + +describe('generated SPEC assessment snapshot', () => { + const original = splitSpecAssessment(readFileSync(path.join(root, assessmentSpecPath), 'utf8')).contract; + const snapshot = specAssessmentFor(assessControl(options()), '0.7.0'); + + for (const eol of ['\n', '\r\n']) { + it(`preserves all contract bytes and lifecycle with ${JSON.stringify(eol)} line endings`, () => { + const spec = original.replace(/\r\n/g, '\n').replace(/\n/g, eol); + const updated = updateSpecAssessment(spec, snapshot); + assert.deepEqual(splitSpecAssessment(updated), { contract: spec, assessment: snapshot }); + assert.equal(updateSpecAssessment(updated, snapshot), updated); + assert.equal((updated.match(/status: implemented/g) ?? []).length, 1); + assert.match(updated, /assessment-stage: 'not-determined'/); + }); + } + + it('keeps the reviewed mapping digest stable only for the generated block', () => { + const read = (spec: string) => + readControlContractFrom((file) => (file === assessmentSpecPath ? spec : readFileSync(path.join(root, file), 'utf8'))); + const generated = updateSpecAssessment(original, snapshot); + assert.equal(read(generated).digest, contract.digest); + assert.equal(read(generated).mappingCurrent, true); + assert.notEqual(read(`${generated}\nNew human contract requirement.\n`).digest, contract.digest); + assert.equal(read(`${generated}\nNew human contract requirement.\n`).mappingCurrent, false); + assert.notEqual(read(generated.replace('status: implemented', 'status: contract-reviewed')).digest, contract.digest); + }); + + it('does not invent a tested commit from missing, dirty, mismatched, or stale evidence', () => { + for (const report of [ + assessControl(options({ report: undefined, context: undefined, reportSha256: undefined })), + assessControl(options({ workingTreeDirty: true })), + assessControl(options({ context: { ...context, sourceCommit: 'b'.repeat(40) } })), + assessControl(options({ maxAgeHours: 0.01 })), + ]) { + const metadata = specAssessmentFor(report, '0.7.0'); + assert.equal(metadata.testedCommit, null); + assert.equal(metadata.sourceCommit, sourceCommit); + assert.equal(metadata.provenance, report.evidence.status); + assert.equal(metadata.health, report.assessmentHealth); + assert.equal(metadata.outcome, report.assessmentOutcome); + assert.match(updateSpecAssessment(original, metadata), /assessment-tested-commit: null/); + } + assert.equal(snapshot.testedCommit, sourceCommit); + }); + + it('preserves native assertion and execution failures rather than promoting a maturity stage', () => { + const failure = { ...direct, status: 'failed', tests: [{ ...nativeCase, status: 'failed', error: 'Native assertion failed.' }] }; + const bytes = JSON.stringify(failure); + const failed = assessControl( + options({ + report: parseNativeReport(failure), + reportSha256: reportDigest(bytes), + context: { ...context, reportSha256: reportDigest(bytes) }, + }), + ); + assert.equal(specAssessmentFor(failed, '0.7.0').outcome, 'failed'); + const blocked = assessControl(options({ executionFailure: 'Native helper exited unexpectedly.' })); + assert.equal(specAssessmentFor(blocked, '0.7.0').outcome, 'execution-failed'); + }); + + it('rejects malformed, duplicated, unknown, misplaced, noncanonical and falsely bound metadata', () => { + const valid = updateSpecAssessment(original, snapshot); + const malformed = [ + original.replace('status: implemented', 'status: implemented\nassessment-outcome: "failed"'), + valid.replace('assessment-schema: 1', 'assessment-schema: 2'), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\nassessment-schema: 1'), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\nassessment-extra: "arbitrary contract content"'), + valid.replace("assessment-outcome: 'insufficient-evidence'", "assessment-outcome: 'production-ready'"), + valid.replace("assessment-stage: 'not-determined'", "assessment-stage: 'beta'"), + valid.replace("assessment-package-version: '0.7.0'", "assessment-package-version: '0.7.0'\ninjected: true"), + valid.replace("assessment-generated-at: '2026-10-21T21:00:00.000Z'", "assessment-generated-at: '2026-02-30T21:00:00.000Z'"), + valid.replace('assessment-dirty: false', 'assessment-dirty: true'), + valid.replace('assessment-tested-commit:', 'assessment-wrong-commit:'), + valid.replace('# furn-assessment:end', ''), + `${valid}\n# furn-assessment:begin\n`, + `${original}\n# furn-assessment:begin\n# furn-assessment:end\n`, + valid.replace('assessment-schema: 1', 'assessment-schema: 1'), + valid.replace("assessment-target: 'win32-paper-storybook'", "assessment-target: 'windows-fabric-storybook'"), + valid.replace("assessment-report: './assessment/ASSESSMENT.md'", "assessment-report: '../other-file.md'"), + valid.replace('assessment-schema: 1', 'assessment-schema: 1\n__proto__: {"assessment-schema":1}'), + ]; + for (const content of malformed) assert.throws(() => splitSpecAssessment(content)); + assert.throws(() => updateSpecAssessment('No front matter', snapshot), /front matter/); + assert.throws(() => updateSpecAssessment('---\nunterminated', snapshot), /unterminated/); + assert.throws(() => updateSpecAssessment(original, { ...snapshot, packageVersion: 'not-a-version' }), /identity/); + assert.throws(() => updateSpecAssessment(original, { ...snapshot, testedCommit: null }), /matched, clean/); + }); +}); diff --git a/scripts/src/ci/assessment/controlAssessment.ts b/scripts/src/ci/assessment/controlAssessment.ts new file mode 100644 index 0000000000..a362b4eee0 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessment.ts @@ -0,0 +1,721 @@ +import { createHash } from 'node:crypto'; +import { readFileSync, realpathSync } from 'node:fs'; +import path from 'node:path'; + +import { splitSpecAssessment } from './controlAssessmentSpec.ts'; +import { assessControlUnits } from './controlAssessmentUnit.ts'; +import type { UnitInput } from './controlAssessmentUnit.ts'; +import { readMappingContent, selectAssessment } from './assessmentConfig.ts'; +import type { AssessmentDefinition } from './assessmentConfig.ts'; + +type Requirement = { id: string; summary: string; declaredEvidence: string[] }; +export type ControlContract = { + definition: AssessmentDefinition; + digest: string; + mappingCurrent: boolean; + lifecycle: string; + conformance: string; + requirements: Requirement[]; +}; +type CaseStatus = 'passed' | 'failed' | 'skipped'; +type NativeCase = { storyId: string; testName: string; status: CaseStatus; durationMs: number; reason?: string }; +export type NativeReport = { + endpoint: 'win32' | 'windows' | 'macos'; + platformManifestDigest: string; + status: 'passed' | 'failed'; + error?: string; + targetId?: string; + startedAt?: string; + finishedAt?: string; + cases: NativeCase[]; + format: 'wdio' | 'storybook-smoke' | 'desktop-test'; + harness?: { + runId: string; + status: string; + retries: number; + flaky: number; + unsupported: number; + incomplete: number; + failureCount: number; + }; +}; +export type EvidenceContext = { + schemaVersion: 1; + reportSha256: string; + sourceCommit: string; + buildId: string; + contractDigest: string; + platformManifestDigest: string; + endpoint: 'win32'; + renderer: 'paper'; + targetId: string; + finishedAt: string; + physicalClick: boolean; +}; +type Provenance = { status: 'missing' | 'unverified' | 'matched' | 'mismatched' | 'stale'; reasons: string[] }; +type CheckOutcome = CaseStatus | 'not-run' | 'unverified' | 'mismatched' | 'stale'; +export type AssessmentOptions = { + contract: ControlContract; + sourceCommit: string; + workingTreeDirty: boolean; + now: Date; + report?: NativeReport; + reportSha256?: string; + context?: EvidenceContext; + expectedBuildId?: string; + expectedTargetId?: string; + maxAgeHours?: number; + executionFailure?: string; + unit?: UnitInput; +}; + +function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function object(value: unknown, label: string): Record { + if (!isObject(value)) throw new Error(`${label} must be an object.`); + return value; +} + +export function text(value: unknown, label: string): string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be a nonempty string.`); + return value; +} + +export function hex(value: unknown, length: number, label: string): string { + const result = text(value, label); + if (!new RegExp(`^[0-9a-f]{${length}}$`).test(result)) throw new Error(`${label} must be ${length} lowercase hex characters.`); + return result; +} + +function choice(value: unknown, choices: readonly T[], label: string): T { + const result = choices.find((entry) => entry === value); + if (result === undefined) throw new Error(`${label} must be one of: ${choices.join(', ')}.`); + return result; +} + +export function timestamp(value: unknown, label: string): string { + const result = text(value, label); + const time = Date.parse(result); + if ( + !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/.test(result) || + !Number.isFinite(time) || + new Date(time).toISOString().slice(0, 19) !== result.slice(0, 19) + ) { + throw new Error(`${label} must be a valid UTC ISO timestamp.`); + } + return result; +} + +export function reportDigest(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + +export function win32StorybookTargetId(root: string): string { + // Mirror desktop-test's target identity protocol without loading PR configuration in the trusted publisher. + const appRoot = realpathSync(path.join(root, 'apps', 'storybook')); + return `win32-${reportDigest(`${appRoot}\0win32\0paper`).slice(0, 16)}`; +} + +function isWin32StorybookTarget(value: string): boolean { + return value === 'agenticstorybook-win32' || /^win32-[0-9a-f]{16}$/.test(value); +} + +export function readControlContract(root: string, definition = selectAssessment(root)): ControlContract { + return readControlContractFrom((file) => readFileSync(path.join(root, file), 'utf8'), definition); +} + +export function readControlContractFrom(read: (file: string) => string, definition: AssessmentDefinition): ControlContract { + const componentPath = `${definition.packagePath}/${definition.componentPath}`; + const files = new Map( + definition.contract.files.map((file) => { + const content = read(`${componentPath}/${file}`).replace(/\r\n/g, '\n'); + return [file, file === 'SPEC.md' ? splitSpecAssessment(content).contract : content]; + }), + ); + const hash = createHash('sha256'); + for (const [file, content] of files) hash.update(`${file}\0${content}\0`); + for (const file of definition.contract.assertionFiles) { + const name = `${definition.packagePath}/${file}`; + hash.update(`${name}\0${read(name).replace(/\r\n/g, '\n')}\0`); + } + hash.update(readMappingContent(read, definition)); + const digest = hash.digest('hex'); + const source = object(JSON.parse(files.get('spec/source.json')!), 'Component source metadata'); + if (source.component !== definition.component || source.schemaVersion !== 2 || !Array.isArray(source.requirements)) { + throw new Error('Expected matching component source metadata schemaVersion 2 and a requirements array.'); + } + const summaries = new Map( + [...files.get('SPEC.md')!.matchAll(/^- \*\*([A-Z][A-Z0-9]*-\d{3}):\*\* (.+(?:\n {2}.+)*)/gm)].map((match) => [ + match[1], + match[2].replace(/\s+/g, ' ').trim(), + ]), + ); + const requirements = source.requirements.map((value) => { + const entry = object(value, 'Requirement'); + const id = text(entry.id, 'Requirement id'); + const summary = summaries.get(id); + if (!summary || !Array.isArray(entry.evidence)) throw new Error(`Missing SPEC requirement or evidence list for ${id}.`); + return { id, summary, declaredEvidence: entry.evidence.map((file) => text(file, `${id} evidence file`)) }; + }); + if (new Set(requirements.map(({ id }) => id)).size !== requirements.length || summaries.size !== requirements.length) { + throw new Error('Component SPEC and metadata must contain the same unique requirement IDs.'); + } + const mappedIds = Object.keys(definition.contract.requirements); + if (mappedIds.length !== requirements.length || requirements.some(({ id }) => !mappedIds.includes(id))) + throw new Error('Assessment matrix must include every SPEC requirement exactly once, with uncovered checks retained.'); + return { + definition, + digest, + mappingCurrent: digest === definition.contract.reviewedDigest, + lifecycle: text(source.lifecycle, 'Contract lifecycle'), + conformance: text(source.conformance, 'Contract conformance'), + requirements, + }; +} + +export function parseNativeReport(value: unknown): NativeReport { + const data = object(value, 'Native report'); + if (data.format === 'desktop-test') return parseDesktopEvidence(data); + const format = 'wdio' in data || 'schemaVersion' in data || 'manifest' in data ? 'storybook-smoke' : 'wdio'; + const entries = format === 'storybook-smoke' ? data.wdio : data.tests; + if (!Array.isArray(entries)) throw new Error('Native report must contain a tests or wdio array.'); + const error = data.error; + if (error !== undefined && typeof error !== 'string') throw new Error('Native report error must be a string.'); + const rawStatus = choice(data.status, ['passed', 'failed', 'skipped', 'incomplete'], 'Report status'); + const status = rawStatus === 'passed' ? 'passed' : 'failed'; + if (status === 'passed' && error !== undefined) throw new Error('Passed report cannot contain a runner error.'); + let platformManifestDigest: string; + let targetId: string | undefined; + let startedAt: string | undefined; + let finishedAt: string | undefined; + if (format === 'storybook-smoke') { + if (data.schemaVersion !== 1 || !Array.isArray(data.tests)) + throw new Error('Storybook smoke report requires schemaVersion 1 and a tests array.'); + const manifest = object(data.manifest, 'Smoke manifest'); + platformManifestDigest = hex(manifest.platform, 64, 'Smoke platform manifest'); + hex(manifest.portable, 64, 'Smoke portable manifest'); + if (data.platformManifestDigest !== undefined && data.platformManifestDigest !== platformManifestDigest) + throw new Error('Conflicting report platform digests.'); + targetId = text(data.targetId, 'Smoke targetId'); + text(data.runId, 'Smoke runId'); + text(data.platformName, 'Smoke platformName'); + finishedAt = timestamp(data.finishedAt, 'Smoke finishedAt'); + startedAt = timestamp(data.startedAt, 'Smoke startedAt'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Smoke report finished before it started.'); + } else { + platformManifestDigest = hex(data.platformManifestDigest, 64, 'Report platformManifestDigest'); + } + const identities = new Set(); + const cases = entries.map((value, index): NativeCase => { + const entry = object(value, `Native case ${index}`); + const id = text(entry.storyId, 'Case storyId'); + const name = entry.testName === undefined ? 'default' : text(entry.testName, 'Case testName'); + const identity = JSON.stringify([id, name]); + if (identities.has(identity)) throw new Error(`Duplicate native case: ${id} / ${name}.`); + identities.add(identity); + const caseStatus = choice(entry.status, ['passed', 'failed', 'skipped'], 'Case status'); + if (typeof entry.durationMs !== 'number' || !Number.isFinite(entry.durationMs) || entry.durationMs < 0) { + throw new Error(`Invalid durationMs for ${id} / ${name}.`); + } + const caseError = entry.error; + if (caseError !== undefined && typeof caseError !== 'string') throw new Error('Case error must be a string.'); + if ( + entry.evidenceErrors !== undefined && + (!Array.isArray(entry.evidenceErrors) || entry.evidenceErrors.some((value) => typeof value !== 'string')) + ) + throw new Error('Case evidenceErrors must be a string array.'); + if (caseStatus === 'passed' && (caseError !== undefined || (Array.isArray(entry.evidenceErrors) && entry.evidenceErrors.length))) + throw new Error('Passed case cannot contain execution or evidence errors.'); + if (status === 'passed' && caseStatus === 'failed') throw new Error('Passed report cannot contain a failed case.'); + return { + storyId: id, + testName: name, + status: caseStatus, + durationMs: entry.durationMs, + reason: caseStatus === 'skipped' ? text(entry.skipReason, 'Skipped case reason') : caseError, + }; + }); + return { + endpoint: choice(data.endpoint, ['win32', 'windows', 'macos'], 'Report endpoint'), + platformManifestDigest, + targetId, + finishedAt, + status, + error: error ?? (rawStatus === 'skipped' || rawStatus === 'incomplete' ? `Native execution is ${rawStatus}.` : undefined), + cases, + format, + startedAt, + }; +} + +const desktopStatuses = ['passed', 'failed', 'skipped', 'unsupported', 'cancelled', 'not-run', 'incomplete'] as const; +const smokeArtifact = 'evidence/win32/static/run.json'; + +export function hasStorybookArtifact(run: Record): boolean { + if (!Array.isArray(run.artifacts)) throw new Error('Desktop supervisor report has no artifact inventory.'); + return run.artifacts.some( + (value) => text(object(value, 'Desktop artifact').path, 'Artifact path').replaceAll('\\', '/') === smokeArtifact, + ); +} + +/** Keep the canonical supervisor result and its Storybook assertions together, including their exact bytes. */ +export function readNativeEvidence(file: string): string { + const content = readFileSync(file, 'utf8'); + const data = object(JSON.parse(content), 'Native evidence'); + if (data.kind !== 'tests' && data.kind !== 'lifecycle') return content; + const root = realpathSync(path.dirname(file)); + const companion = realpathSync(path.join(root, ...smokeArtifact.split('/'))); + const relative = path.relative(root, companion); + if (relative.startsWith('..') || path.isAbsolute(relative)) throw new Error('Storybook evidence must remain inside its desktop run.'); + return JSON.stringify({ schemaVersion: 1, format: 'desktop-test', desktopRun: content, storyReport: readFileSync(companion, 'utf8') }); +} + +function parseDesktopEvidence(data: Record): NativeReport { + if (data.schemaVersion !== 1) throw new Error('Unsupported desktop evidence envelope.'); + const run = object(JSON.parse(text(data.desktopRun, 'Desktop run bytes')), 'Desktop run'); + if (run.schemaVersion !== 1 || run.kind !== 'tests') throw new Error('Expected a desktop-test schemaVersion 1 test run.'); + const runId = text(run.runId, 'Desktop run ID'); + const startedAt = timestamp(run.startedAt, 'Desktop startedAt'); + const finishedAt = timestamp(run.finishedAt, 'Desktop finishedAt'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Desktop run finished before it started.'); + const status = choice(run.status, desktopStatuses, 'Desktop status'); + if (!Array.isArray(run.tests) || !Array.isArray(run.work) || !Array.isArray(run.failures) || !Array.isArray(run.artifacts)) { + throw new Error('Desktop run requires tests, work, failures and artifacts arrays.'); + } + const artifactMatches = run.artifacts.filter( + (value) => text(object(value, 'Desktop artifact').path, 'Artifact path').replaceAll('\\', '/') === smokeArtifact, + ); + if (artifactMatches.length !== 1) throw new Error('Desktop run must declare exactly one Win32 Storybook smoke artifact.'); + const storyData = object(JSON.parse(text(data.storyReport, 'Storybook report bytes')), 'Storybook smoke evidence'); + if (!('manifest' in storyData) || storyData.format === 'desktop-test') throw new Error('Expected a Storybook smoke companion.'); + const report = parseNativeReport(storyData); + if (report.format !== 'storybook-smoke' || report.endpoint !== 'win32') throw new Error('Expected Win32 Storybook smoke evidence.'); + if (Date.parse(report.startedAt!) < Date.parse(startedAt) || Date.parse(report.finishedAt!) > Date.parse(finishedAt)) { + throw new Error('Storybook evidence is outside its enclosing desktop execution.'); + } + const counts = { selected: 0, executed: 0, passed: 0, failed: 0, skipped: 0, unsupported: 0, incomplete: 0, retries: 0, flaky: 0 }; + const cases = new Map(); + const ids = new Set(); + for (const value of run.tests) { + const test = object(value, 'Desktop test'); + const id = text(test.testId, 'Desktop test ID'); + if (ids.has(id)) throw new Error('Duplicate desktop test ID.'); + ids.add(id); + text(test.title, 'Desktop test title'); + const testStatus = choice(test.status, desktopStatuses, 'Desktop test status'); + if (!Array.isArray(test.attempts) || !test.attempts.length) throw new Error('Desktop test needs recorded attempts.'); + let previous = -1; + let failedAttempt = false; + let executed = false; + let identity: string | undefined; + const attempts = test.attempts.map((value) => { + const attempt = object(value, 'Desktop attempt'); + if (!Number.isSafeInteger(attempt.attempt) || Number(attempt.attempt) <= previous) throw new Error('Invalid desktop attempt order.'); + previous = Number(attempt.attempt); + const result = choice(attempt.status, desktopStatuses, 'Desktop attempt status'); + if (typeof attempt.durationMs !== 'number' || !Number.isFinite(attempt.durationMs) || attempt.durationMs < 0) { + throw new Error('Invalid desktop attempt duration.'); + } + if (!['passed', 'failed'].includes(result)) text(attempt.reason, 'Desktop nonexecution reason'); + if (result === 'failed') object(attempt.failure, 'Desktop failed attempt'); + if (result === 'passed' && attempt.failure !== undefined) throw new Error('Passed desktop attempt contains a failure.'); + failedAttempt ||= result === 'failed'; + executed ||= result === 'passed' || result === 'failed'; + if (attempt.context !== undefined) { + const context = object(attempt.context, 'Desktop attempt context'); + if (id.startsWith('csf-')) { + const key = JSON.stringify([text(context.story, 'Desktop story'), text(context.test, 'Desktop test name')]); + if (identity !== undefined && identity !== key) throw new Error('Desktop test identity changed across attempts.'); + identity = key; + if (id !== `csf-${reportDigest(key)}`) throw new Error('Desktop test ID does not match the authored callback identity.'); + if (context.endpoint !== report.endpoint) throw new Error('Desktop attempt endpoint differs from Storybook evidence.'); + } + } + return result; + }); + if (attempts.at(-1) !== testStatus) throw new Error('Desktop final status disagrees with its last attempt.'); + const flaky = testStatus === 'passed' && failedAttempt; + if (test.flaky !== flaky) throw new Error('Desktop flaky flag disagrees with recorded attempts.'); + if (identity !== undefined) { + if (cases.has(identity)) throw new Error('Duplicate desktop story/test identity.'); + cases.set(identity, testStatus); + } + counts.selected++; + counts.executed += Number(executed); + counts.retries += test.attempts.length - 1; + counts.flaky += Number(flaky); + if (testStatus === 'passed' || testStatus === 'failed' || testStatus === 'skipped' || testStatus === 'unsupported') + counts[testStatus]++; + else counts.incomplete++; + } + const declaredCounts = object(run.counts, 'Desktop counts'); + for (const [name, value] of Object.entries(counts)) { + if (declaredCounts[name] !== value) throw new Error(`Desktop ${name} count disagrees with recorded attempts.`); + } + const work = run.work.map((value) => choice(object(value, 'Desktop selected work').status, desktopStatuses, 'Desktop work status')); + let infrastructureFailed = false; + for (const value of run.failures) { + const failure = object(value, 'Desktop failure'); + text(failure.failureId, 'Desktop failure ID'); + const kind = choice(failure.kind, ['test', 'infrastructure', 'cleanup', 'evidence', 'reporting'], 'Desktop failure kind'); + if (typeof object(failure.error, 'Desktop error').message !== 'string') throw new Error('Desktop error message must be a string.'); + infrastructureFailed ||= kind !== 'test'; + } + if ( + status === 'passed' && + (infrastructureFailed || + counts.failed || + counts.incomplete || + !counts.executed || + work.some((value) => ['failed', 'not-run', 'incomplete', 'cancelled'].includes(value))) + ) + throw new Error('Passed desktop run contains failures or incomplete execution.'); + for (const entry of report.cases) { + const recorded = cases.get(JSON.stringify([entry.storyId, entry.testName])); + if (recorded !== entry.status) throw new Error('Storybook case does not match canonical desktop results.'); + } + const issues = [ + ...(report.error ? [report.error] : []), + ...(status !== 'passed' ? [`Desktop supervisor status: ${status}.`] : []), + ...(run.failures.length ? [`Desktop run recorded ${run.failures.length} failure(s), including prior attempts.`] : []), + ...(counts.retries || counts.flaky ? ['Retried or flaky desktop execution is not qualifying evidence in this pilot.'] : []), + ]; + return { + ...report, + format: 'desktop-test', + startedAt, + finishedAt, + status: status === 'passed' && report.status === 'passed' && !issues.length ? 'passed' : 'failed', + error: issues.length ? issues.join(' ') : undefined, + harness: { + runId, + status, + retries: counts.retries, + flaky: counts.flaky, + unsupported: counts.unsupported, + incomplete: counts.incomplete, + failureCount: run.failures.length, + }, + }; +} + +export function parseEvidenceContext(value: unknown): EvidenceContext { + const data = object(value, 'Evidence context'); + if (data.schemaVersion !== 1) throw new Error('Evidence context schemaVersion must be 1.'); + if (typeof data.physicalClick !== 'boolean') throw new Error('Evidence context physicalClick must be boolean.'); + const targetId = text(data.targetId, 'Context targetId'); + if (!isWin32StorybookTarget(targetId)) throw new Error('Context targetId is not a supported Win32 Storybook target.'); + return { + schemaVersion: 1, + reportSha256: hex(data.reportSha256, 64, 'Context reportSha256'), + sourceCommit: hex(data.sourceCommit, 40, 'Context sourceCommit'), + buildId: text(data.buildId, 'Context buildId'), + contractDigest: hex(data.contractDigest, 64, 'Context contractDigest'), + platformManifestDigest: hex(data.platformManifestDigest, 64, 'Context platformManifestDigest'), + endpoint: choice(data.endpoint, ['win32'], 'Context endpoint'), + renderer: choice(data.renderer, ['paper'], 'Context renderer'), + targetId, + finishedAt: timestamp(data.finishedAt, 'Context finishedAt'), + physicalClick: data.physicalClick, + }; +} + +function provenance(options: AssessmentOptions): Provenance { + const { report, context, contract } = options; + const { storyId, testName, requiresPhysicalClick } = contract.definition.contract.nativeCheck; + if (!report) return { status: 'missing', reasons: ['No native execution report supplied.'] }; + const mismatches: string[] = []; + if (report.endpoint !== 'win32') mismatches.push(`Report endpoint is ${report.endpoint}, not win32.`); + if (report.targetId !== undefined && !isWin32StorybookTarget(report.targetId)) mismatches.push('Report targets a different native host.'); + if (report.targetId !== undefined && options.expectedTargetId !== undefined && report.targetId !== options.expectedTargetId) + mismatches.push('Report target does not match the expected native instance.'); + if (context) { + if (context.reportSha256 !== options.reportSha256) mismatches.push('Execution context is bound to different report bytes.'); + if (context.sourceCommit !== options.sourceCommit) mismatches.push('Execution source revision does not match the target revision.'); + if (context.contractDigest !== contract.digest) mismatches.push('Execution contract/mapping inputs do not match the current inputs.'); + if (context.platformManifestDigest !== report.platformManifestDigest) + mismatches.push('Execution platform manifest does not match the report.'); + if (report.targetId !== undefined && context.targetId !== report.targetId) + mismatches.push('Execution target does not match the report.'); + if (options.expectedTargetId !== undefined && context.targetId !== options.expectedTargetId) + mismatches.push('Execution target does not match the expected native instance.'); + if (report.finishedAt !== undefined && Date.parse(report.finishedAt) !== Date.parse(context.finishedAt)) + mismatches.push('Execution timestamp does not match the report.'); + if (options.expectedBuildId !== undefined && context.buildId !== options.expectedBuildId) + mismatches.push('Execution build ID does not match the requested build.'); + if (Date.parse(context.finishedAt) > options.now.getTime()) mismatches.push('Execution timestamp is in the future.'); + if ( + requiresPhysicalClick && + !context.physicalClick && + report.cases.some((entry) => entry.storyId === storyId && entry.testName === testName && entry.status === 'passed') + ) { + mismatches.push('Recorded native pass contradicts the execution physical-click capability.'); + } + } + if (mismatches.length) return { status: 'mismatched', reasons: mismatches }; + if ( + context && + options.maxAgeHours !== undefined && + options.now.getTime() - Date.parse(context.finishedAt) > options.maxAgeHours * 3_600_000 + ) { + return { status: 'stale', reasons: ['Execution evidence exceeds the explicitly requested maximum age.'] }; + } + const reasons: string[] = []; + if (!context) reasons.push('Report has no bound execution context; source/build/freshness cannot be established.'); + if (context?.targetId !== 'agenticstorybook-win32' && context && options.expectedTargetId === undefined) + reasons.push('No expected native target instance was supplied.'); + if (options.workingTreeDirty) reasons.push('Target working tree has uncommitted changes; a commit alone does not identify these inputs.'); + if (options.expectedBuildId === undefined) reasons.push('No expected build ID was supplied.'); + if (options.maxAgeHours === undefined) reasons.push('No evidence-age policy was supplied.'); + return { status: reasons.length ? 'unverified' : 'matched', reasons }; +} + +export function assessControl(options: AssessmentOptions) { + const definition = options.contract.definition; + const { storyId, testName } = definition.contract.nativeCheck; + hex(options.sourceCommit, 40, 'Target sourceCommit'); + if (!Number.isFinite(options.now.getTime())) throw new Error('Assessment time is invalid.'); + if (options.maxAgeHours !== undefined && (!Number.isFinite(options.maxAgeHours) || options.maxAgeHours <= 0)) { + throw new Error('Maximum evidence age must be a positive finite number.'); + } + if (options.context && !options.report) throw new Error('Execution context requires a native report.'); + if (options.report) hex(options.reportSha256, 64, 'Supplied report content hash'); + if (options.expectedBuildId !== undefined) text(options.expectedBuildId, 'Expected build ID'); + if (options.expectedTargetId !== undefined && !isWin32StorybookTarget(options.expectedTargetId)) + throw new Error('Expected target ID is not a supported Win32 Storybook target.'); + const evidence = provenance(options); + const mappingCurrent = options.contract.mappingCurrent; + const unitEvidence = assessControlUnits({ + definition, + input: options.unit, + sourceCommit: options.sourceCommit, + workingTreeDirty: options.workingTreeDirty, + contractDigest: options.contract.digest, + contractMappingCurrent: mappingCurrent, + now: options.now, + maxAgeHours: options.maxAgeHours, + }); + const runFailed = options.report?.status === 'failed' || Boolean(options.report?.error) || Boolean(options.executionFailure); + const observed = options.report?.cases.find((entry) => entry.storyId === storyId && entry.testName === testName); + const eligible = evidence.status === 'matched' && mappingCurrent; + const requirements = options.contract.requirements.map((requirement) => { + const mapping = definition.contract.requirements[requirement.id]; + const mapped = Boolean(mapping?.observed); + let outcome: CheckOutcome = 'not-run'; + if (mapped && observed) { + if (!mappingCurrent || evidence.status === 'stale') outcome = 'stale'; + else if (evidence.status === 'mismatched') outcome = 'mismatched'; + else if (!eligible) outcome = 'unverified'; + else if (observed.status === 'passed' && runFailed) outcome = 'unverified'; + else outcome = observed.status; + } + const check = mapped + ? { + storyId, + testName, + scope: mapping.observed!, + coverage: 'partial' as const, + recordedOutcome: observed?.status ?? 'not-run', + evidenceOutcome: outcome, + reason: observed?.reason, + } + : null; + const unitChecks = unitEvidence.checks.filter(({ requirements }) => requirements.includes(requirement.id)); + const checks = (mapping?.checks ?? []).map((specific) => { + const tests = specific.tests.map((test) => { + if (test.kind === 'native') + return { + ...test, + recordedOutcome: observed?.status ?? 'not-run', + evidenceOutcome: outcome, + reason: observed?.reason ?? null, + }; + const result = unitEvidence.checks.find(({ fullName }) => fullName === test.fullName)!; + return { + ...test, + recordedOutcome: result.recordedOutcome, + evidenceOutcome: result.evidenceOutcome, + reason: result.reason, + }; + }); + const outcomes = tests.map(({ evidenceOutcome }) => evidenceOutcome); + return { + ...specific, + tests, + outcome: !tests.length + ? 'uncovered' + : outcomes.includes('failed') + ? 'failed' + : outcomes.every((value) => value === 'passed') + ? 'passed' + : outcomes.find((value) => value !== 'passed')!, + }; + }); + const failed = outcome === 'failed' || unitChecks.some(({ evidenceOutcome }) => evidenceOutcome === 'failed'); + const partial = outcome === 'passed' || unitChecks.some(({ evidenceOutcome }) => evidenceOutcome === 'passed'); + return { + ...requirement, + outcome: failed ? 'blocked-by-failure' : partial ? 'partial' : 'insufficient-evidence', + check, + unitChecks, + checks, + remainingCoverage: mapping?.remaining ?? 'No reviewed executable mapping exists for this requirement.', + }; + }); + return { + schemaVersion: 2, + component: definition.component, + displayName: definition.displayName, + generation: 'agentic', + target: { + endpoint: 'win32', + renderer: 'paper', + host: 'storybook', + sourceCommit: options.sourceCommit, + workingTreeDirty: options.workingTreeDirty, + expectedBuildId: options.expectedBuildId ?? null, + }, + generatedAt: options.now.toISOString(), + advisory: true, + declaredStatus: null, + evidenceSupportedStatus: null, + approvedStatus: null, + policyStatus: 'pending', + assessmentHealth: + !mappingCurrent || evidence.status === 'stale' || unitEvidence.status === 'stale' || unitEvidence.mappingCurrent === false + ? 'stale' + : evidence.status === 'mismatched' || unitEvidence.status === 'mismatched' + ? 'blocked' + : 'complete', + assessmentOutcome: + requirements.some(({ outcome }) => outcome === 'blocked-by-failure') || unitEvidence.outcome === 'failed' + ? 'failed' + : (eligible && runFailed) || unitEvidence.outcome === 'execution-failed' + ? 'execution-failed' + : 'insufficient-evidence', + contract: { + path: definition.specPath, + digest: options.contract.digest, + lifecycle: options.contract.lifecycle, + conformance: options.contract.conformance, + mappingVersion: definition.contract.mappingVersion, + mappingCurrent, + }, + evidence: { + ...evidence, + reportSha256: options.reportSha256 ?? null, + platformManifestDigest: options.report?.platformManifestDigest ?? null, + runStatus: options.report?.status ?? null, + runError: options.executionFailure ?? options.report?.error ?? null, + finishedAt: options.report?.finishedAt ?? null, + format: options.report?.format ?? null, + harness: options.report?.harness ?? null, + context: options.context ?? null, + caseCounts: { + passed: options.report?.cases.filter(({ status }) => status === 'passed').length ?? 0, + failed: options.report?.cases.filter(({ status }) => status === 'failed').length ?? 0, + skipped: options.report?.cases.filter(({ status }) => status === 'skipped').length ?? 0, + }, + }, + unitEvidence, + requirements, + limitations: [ + ...definition.limitations, + 'Unit evidence is mocked JavaScript execution, not native Win32 UIA, focus, visual, or physical-input qualification.', + 'Evidence requirements follow the claim: unit/type checks can suffice for logic/API/style requirements; native tests are not mandatory duplicates for every requirement.', + 'Type/build/visual/manual result adapters are not implemented. A Jest pass of a types test would not prove type checking.', + 'Maturity gates and release approvals are not defined by this pilot. No lifecycle status is inferred or changed.', + 'A multi-assertion test failure blocks every check linked to that test; it does not identify which individual assertion failed.', + ...(!mappingCurrent + ? ['Contract or story inputs changed since mapping review; review the mapping before accepting its evidence.'] + : []), + ...(runFailed + ? ['The enclosing execution failed or reported an error; recorded passes are not accepted as qualifying evidence.'] + : []), + ...(unitEvidence.runFailed ? ['The enclosing unit command failed; its recorded passes cannot qualify.'] : []), + ], + }; +} + +export type ControlAssessment = ReturnType; + +function markdown(value: string): string { + return value.replace(/[\\|`*_[\]<>]/g, '\\$&').replace(/[\r\n]+/g, ' '); +} + +export function formatAssessmentMarkdown(report: ControlAssessment): string { + return [ + `# ${markdown(report.displayName)} / Win32 advisory assessment`, + '', + '**Readiness: not established.** Maturity policy is pending; no stage is assigned.', + '', + `Assessment health: ${report.assessmentHealth}. Outcome: ${report.assessmentOutcome}.`, + `Target source: \`${report.target.sourceCommit}\`. Working tree dirty: ${report.target.workingTreeDirty}.`, + `Native execution provenance: ${report.evidence.status}. Mapping current: ${report.contract.mappingCurrent}.`, + `Recorded native cases across the input report: ${report.evidence.caseCounts.passed} passed, ${report.evidence.caseCounts.failed} failed, ${report.evidence.caseCounts.skipped} skipped.`, + ...(report.evidence.harness + ? [ + `Desktop supervisor: ${report.evidence.harness.status}; retries: ${report.evidence.harness.retries}; flaky: ${report.evidence.harness.flaky}; unsupported: ${report.evidence.harness.unsupported}; incomplete: ${report.evidence.harness.incomplete}; failures: ${report.evidence.harness.failureCount}.`, + ] + : []), + '', + `Unit execution provenance: ${report.unitEvidence.status}. Outcome: ${report.unitEvidence.outcome}. Mapping current: ${report.unitEvidence.mappingCurrent ?? 'not collected'}.`, + `Unit environment: ${report.unitEvidence.environment}.`, + `Recorded unit cases: ${report.unitEvidence.caseCounts.passed} passed, ${report.unitEvidence.caseCounts.failed} failed, ${report.unitEvidence.caseCounts.skipped} skipped.`, + `Recorded unit retries: ${report.unitEvidence.retryCounts.retries}; retry-success cases: ${report.unitEvidence.retryCounts.flaky}. Enclosing package retries: ${report.unitEvidence.packageRetryCounts?.retries ?? 0}.`, + ...(report.unitEvidence.packageSuites !== null + ? [ + `Unit input contained ${report.unitEvidence.packageSuites} package suites; only the configured component runtime suites are mapped. Type-suite Jest passes are not type-check evidence.`, + ] + : []), + '', + '| Requirement | Evidence outcome | Recorded native case | Qualifying partial unit cases | Remaining coverage |', + '| --- | --- | --- | --- | --- |', + ...report.requirements.map( + (entry) => + `| ${markdown(entry.id)} | ${entry.outcome} | ${entry.check ? `${entry.check.recordedOutcome} (partial mapping; ${entry.check.evidenceOutcome})` : 'No executable mapping'} | ${entry.unitChecks.filter(({ evidenceOutcome }) => evidenceOutcome === 'passed').length} / ${entry.unitChecks.length} mapped | ${markdown(entry.remainingCoverage)} |`, + ), + '', + '## Test matrix', + '', + 'Each row is a contract-derived acceptance check. All linked tests must qualify to pass a row; uncovered rows remain visible. Passing rows do not establish complete requirement coverage or a maturity stage.', + '', + '| SPEC requirement | Specific check | Contract source | Required evidence | Test(s) | Check outcome |', + '| --- | --- | --- | --- | --- | --- |', + ...report.requirements.flatMap((requirement) => + requirement.checks.map( + (check) => + `| ${markdown(requirement.id)} | ${markdown(check.id)}: ${markdown(check.description)} | ${markdown(check.source)} | ${check.evidence} | ${check.tests.map((test) => `${markdown(test.kind === 'unit' ? test.fullName : `${test.storyId} / ${test.testName}`)} (recorded: ${test.recordedOutcome}; evidence: ${test.evidenceOutcome})`).join('; ') || 'No reviewed test mapping'} | ${check.outcome} |`, + ), + ), + '', + '## Execution evidence', + '', + ...report.requirements + .filter(({ check }) => check) + .map( + (entry) => + `- ${entry.id}: ${markdown(entry.check!.scope)} Recorded: ${entry.check!.recordedOutcome}.${entry.check!.reason ? ` Reason: ${markdown(entry.check!.reason!)}` : ''}`, + ), + ...report.evidence.reasons.map((reason) => `- ${markdown(reason)}`), + '', + '## JavaScript unit evidence', + '', + ...report.unitEvidence.reasons.map((reason) => `- ${markdown(reason)}`), + ...(report.unitEvidence.executionFailure ? [`- ${markdown(report.unitEvidence.executionFailure)}`] : []), + '| Test | Win32 requirement mapping | Recorded | Evidence outcome | Retries | Scope |', + '| --- | --- | --- | --- | --- | --- |', + ...report.unitEvidence.checks.map( + (check) => + `| ${markdown(check.fullName)} | ${check.requirements.join(', ') || 'Not mapped to Win32'} | ${check.recordedOutcome} | ${check.evidenceOutcome} | ${check.retries} | ${markdown(check.scope)}${check.reason ? ` ${markdown(check.reason)}` : ''}${check.retryReasons.length ? ` Prior retry failures: ${markdown(check.retryReasons.join('\n'))}` : ''} |`, + ), + ...report.unitEvidence.unmappedCases.map((check) => `- Unmapped recorded unit case: ${markdown(check.fullName)} (${check.status}).`), + '', + '## Limitations', + '', + ...report.limitations.map((limitation) => `- ${markdown(limitation)}`), + '', + ].join('\n'); +} diff --git a/scripts/src/ci/assessment/controlAssessmentCi.test.ts b/scripts/src/ci/assessment/controlAssessmentCi.test.ts new file mode 100644 index 0000000000..f04433ad12 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentCi.test.ts @@ -0,0 +1,1212 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + assertFresh, + assessCiBundle, + beginCiExecution, + combineCiEvidence, + finishCiExecution, + formatCiSummary, + formatCiSpec, + formatCiMapping, + git, + inputDigest, + isReportOnlyUpdate, + parseBundle, + parseIdentity, + parseReportReference, +} from './controlAssessmentCi.ts'; +import type { CiBundle } from './controlAssessmentCi.ts'; +import { readControlContract, reportDigest, win32StorybookTargetId } from './controlAssessment.ts'; +import { splitSpecAssessment } from './controlAssessmentSpec.ts'; +import { + assessmentDefinition, + assessmentMappingPath, + assessmentPackagePath, + assessmentSpecPath, + assessmentSummaryPath, + checkboxNativeAssertionPath, + checkboxJestFixture, + componentJestFixture, + desktopEvidenceFixture, + readUnitMappingFrom, + unitMappingFiles, +} from './controlAssessmentFixtures.ts'; +import { runJestWithUnitCapture } from './controlAssessmentUnitExecution.ts'; +import { + AssessmentGitHub, + inspectReportUpdate, + inspectReportUpdates, + preparePublication, + preparePublications, + publishAssessment, + publishAssessments, + verifyReportUpdate, + verifyReportUpdates, + readBundles, +} from './controlAssessmentGithub.ts'; +import { formatAssessmentDocument } from './assessmentMarkdown.ts'; +import { discoverAssessments } from './assessmentConfig.ts'; + +const repositoryRoot = fileURLToPath(new URL('../../../../', import.meta.url)); +const contractPath = 'packages/agentic/components/src/components/checkbox'; +const now = new Date('2026-10-21T20:00:03.000Z'); +const directories: string[] = []; +const reportDirectory = [ + 'apps', + 'storybook', + 'artifacts', + 'win32', + 'assessment-ci', + assessmentDefinition.packagePath, + assessmentDefinition.component, +]; +const desktopDirectory = ['apps', 'storybook', 'artifacts', 'desktop']; +const nativePath = [...desktopDirectory, 'synthetic-desktop-run', 'run.json']; + +function put(root: string, file: string, content: string): void { + const target = path.join(root, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, content); +} + +function commit(root: string, message: string): string { + git(root, ['add', '.']); + git(root, ['commit', '-qm', message]); + return git(root, ['rev-parse', 'HEAD']); +} + +function merge(root: string, base: string, head: string): string { + git(root, ['checkout', '-q', '--detach', base]); + git(root, ['merge', '--no-ff', '-qm', 'Synthetic PR merge', head]); + return git(root, ['rev-parse', 'HEAD']); +} + +function fixture(extraControl = false) { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-ci-pilot-')); + directories.push(directory); + const root = path.join(directory, 'repo'); + mkdirSync(root); + mkdirSync(path.join(root, 'apps', 'storybook'), { recursive: true }); + git(root, ['init', '-q', '-b', 'main']); + git(root, ['config', 'user.name', 'Fixture']); + git(root, ['config', 'user.email', 'fixture@example.invalid']); + git(root, ['config', 'core.autocrlf', 'false']); + git(root, ['config', 'commit.gpgsign', 'false']); + put(root, '.gitignore', 'apps/storybook/artifacts/\n'); + const originalManifest = JSON.parse(readFileSync(path.join(repositoryRoot, assessmentPackagePath), 'utf8')); + put( + root, + assessmentPackagePath, + JSON.stringify({ + name: originalManifest.name, + version: '0.7.0', + furn: originalManifest.furn, + scripts: { test: originalManifest.scripts.test }, + }), + ); + for (const file of unitMappingFiles) put(root, file, readFileSync(path.join(repositoryRoot, file), 'utf8')); + put(root, assessmentMappingPath, readFileSync(path.join(repositoryRoot, assessmentMappingPath), 'utf8')); + put(root, checkboxNativeAssertionPath, readFileSync(path.join(repositoryRoot, checkboxNativeAssertionPath), 'utf8')); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const target = path.join(root, contractPath, file); + mkdirSync(path.dirname(target), { recursive: true }); + if (file === 'SPEC.md') { + writeFileSync(target, splitSpecAssessment(readFileSync(path.join(repositoryRoot, contractPath, file), 'utf8')).contract); + } else { + copyFileSync(path.join(repositoryRoot, contractPath, file), target); + } + } + if (extraControl) { + const mapping = JSON.parse(readFileSync(path.join(root, assessmentMappingPath), 'utf8')); + mapping.component = 'toggle'; + mapping.displayName = 'Toggle'; + mapping.componentPath = 'src/components/toggle'; + mapping.contract.requirements = Object.fromEntries( + Object.entries(mapping.contract.requirements).map(([id, value]) => [id.replace('CBX-', 'TGL-'), value]), + ); + for (const file of mapping.contract.files) { + put( + root, + `packages/agentic/components/src/components/toggle/${file}`, + readFileSync(path.join(root, contractPath, file), 'utf8') + .replaceAll('CBX-', 'TGL-') + .replaceAll('checkbox', 'toggle') + .replaceAll('Checkbox', 'Toggle'), + ); + } + const file = 'packages/agentic/components/src/components/toggle/assessment.mapping.json'; + put(root, file, JSON.stringify(mapping)); + const manifest = JSON.parse(readFileSync(path.join(root, assessmentPackagePath), 'utf8')); + manifest.furn.assessments.push('src/components/toggle/assessment.mapping.json'); + put(root, assessmentPackagePath, JSON.stringify(manifest)); + const definition = discoverAssessments(root).find(({ component }) => component === 'toggle')!; + mapping.contract.reviewedDigest = readControlContract(root, definition).digest; + put(root, file, JSON.stringify(mapping)); + } + put(root, 'source.ts', 'export const value = 0;\n'); + const baseCommit = commit(root, 'Synthetic base'); + git(root, ['checkout', '-qb', 'pilot/checkbox']); + put(root, 'source.ts', 'export const value = 1;\n'); + const headCommit = commit(root, 'Synthetic control change'); + const testedCommit = merge(root, baseCommit, headCommit); + const event = path.join(directory, 'event.json'); + const payload = (head = headCommit, base = baseCommit) => ({ + pull_request: { + number: 123, + head: { sha: head, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + base: { sha: base, ref: 'main', repo: { full_name: 'example/furn' } }, + }, + }); + writeFileSync(event, JSON.stringify(payload())); + const env: NodeJS.ProcessEnv = { + GITHUB_EVENT_PATH: event, + GITHUB_REPOSITORY: 'example/furn', + GITHUB_SHA: testedCommit, + GITHUB_RUN_ID: '456', + GITHUB_RUN_ATTEMPT: '1', + FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES: 'keyboard,physicalClick,wheel', + SMOKE_OUTCOME: 'success', + }; + return { directory, root, baseCommit, headCommit, testedCommit, event, payload, env }; +} +type Fixture = ReturnType; + +function nativeReport(overrides: Record = {}): string { + return JSON.stringify({ + schemaVersion: 1, + endpoint: 'win32', + platformName: 'windows', + targetId: 'agenticstorybook-win32', + runId: 'synthetic-native-run', + startedAt: '2026-10-21T20:00:01.000Z', + finishedAt: '2026-10-21T20:00:02.000Z', + manifest: { platform: 'b'.repeat(64), portable: 'c'.repeat(64) }, + status: 'passed', + tests: [], + wdio: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'skipped', + durationMs: 0, + skipReason: 'This test requires physical pointer input.', + }, + ], + ...overrides, + }); +} + +function execute(f: Fixture): CiBundle { + rmSync(path.join(f.root, ...desktopDirectory, 'synthetic-desktop-run'), { recursive: true, force: true }); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + return finishCiExecution(f.root, f.env, now); +} + +function putNative(root: string, smoke: string): void { + mkdirSync(path.join(root, 'apps', 'storybook'), { recursive: true }); + const capture = path.join(root, ...reportDirectory, 'execution-start.json'); + const targetId = existsSync(capture) ? JSON.parse(readFileSync(capture, 'utf8')).targetId : win32StorybookTargetId(root); + const boundSmoke = JSON.stringify({ ...JSON.parse(smoke), targetId }); + put(root, path.join(...nativePath), desktopEvidenceFixture(boundSmoke).canonical); + put(root, path.join(...desktopDirectory, 'synthetic-desktop-run', 'evidence', 'win32', 'static', 'run.json'), boundSmoke); +} + +function putUnit(f: Fixture, modify: (capture: Record) => void = () => {}): void { + const started = JSON.parse(readFileSync(path.join(f.root, ...reportDirectory, 'execution-start.json'), 'utf8')); + const report = JSON.stringify(checkboxJestFixture()); + const mapping = readUnitMappingFrom((file) => readFileSync(path.join(f.root, file), 'utf8')); + const capture = { + schemaVersion: 1, + identity: started.identity, + exitCode: 0, + report, + executionFailure: null, + collectionStartedAt: '2026-10-21T20:00:00.000Z', + context: { + schemaVersion: 1, + platform: 'macos', + sourceCommit: f.testedCommit, + workingTreeDirty: false, + buildId: 'example/furn/actions/runs/456/attempts/1', + contractDigest: readControlContract(f.root).digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt: '2026-10-21T20:00:00.100Z', + finishedAt: '2026-10-21T20:00:01.500Z', + exitCode: 0, + }, + }; + modify(capture); + put(f.root, path.join(...reportDirectory, 'unit-execution.json'), JSON.stringify(capture)); + f.env.UNIT_OUTCOME = 'success'; +} + +function reportCommit( + f: Fixture, + summary: string, + spec = readFileSync(path.join(f.root, ...reportDirectory, 'SPEC.md'), 'utf8'), +): { head: string; merged: string } { + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, summary); + put(f.root, assessmentSpecPath, spec); + const head = commit(f.root, 'Synthetic generated report'); + const merged = merge(f.root, f.baseCommit, head); + writeFileSync(f.event, JSON.stringify(f.payload(head, f.baseCommit))); + f.env.GITHUB_SHA = merged; + return { head, merged }; +} + +function fakeApi(f: Fixture, overrides: Record = {}) { + const calls: { url: string; options?: RequestInit }[] = []; + const fetchImplementation: typeof fetch = async (input, options) => { + const url = String(input); + calls.push({ url, options }); + const route = url.slice('https://api.github.com'.length); + const defaults: Record = { + '/repos/example/furn/actions/runs/456/attempts/1': { + id: 456, + run_attempt: 1, + event: 'pull_request', + status: 'completed', + path: '.github/workflows/pr.yml', + head_sha: f.headCommit, + head_branch: 'pilot/checkbox', + head_repository: { full_name: 'example/furn' }, + }, + '/repos/example/furn/actions/runs/456/attempts/1/jobs?per_page=100&page=1': { + total_count: 1, + jobs: [{ name: 'PR source validation', conclusion: 'success' }], + }, + '/repos/example/furn/actions/runs/456/artifacts?per_page=100&page=1': { + total_count: 1, + artifacts: [{ name: 'component-assessments-1', expired: false }], + }, + '/repos/example/furn/pulls/123': { + number: 123, + state: 'open', + base: { sha: f.baseCommit, ref: 'main', repo: { full_name: 'example/furn' } }, + head: { sha: f.headCommit, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + merge_commit_sha: f.testedCommit, + }, + '/graphql': { data: { createCommitOnBranch: { commit: { oid: 'd'.repeat(40) } } } }, + }; + const value = route in overrides ? overrides[route] : defaults[route]; + assert.notEqual(value, undefined, `Unexpected API request: ${route}`); + return new Response(JSON.stringify(value), { status: 200 }); + }; + return { github: new AssessmentGitHub({ token: 'synthetic-token', fetchImplementation }), calls }; +} + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('CI execution binding', () => { + it('includes the exact reviewed requirement/check/test mapping in artifacts and rejects a substituted mapping', () => { + const f = fixture(); + const bundle = execute(f); + const directory = path.join(f.root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment-ci'); + const mappingPath = path.join(f.root, ...reportDirectory, 'assessment.mapping.json'); + assert.equal(readFileSync(mappingPath, 'utf8'), readFileSync(path.join(f.root, assessmentMappingPath), 'utf8')); + assert.deepEqual(readBundles(f.root, directory), [bundle]); + const mapping = JSON.parse(readFileSync(mappingPath, 'utf8')); + mapping.contract.requirements['CBX-001'].checks[0].description = 'Unreviewed replacement claim'; + writeFileSync(mappingPath, JSON.stringify(mapping)); + assert.throws(() => readBundles(f.root, directory), /Artifact mapping differs/); + rmSync(mappingPath); + assert.throws(() => readBundles(f.root, directory), /ENOENT/); + }); + + it('assesses existing units when native execution is unavailable and rejects a different native attempt', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + put( + f.root, + path.join( + 'apps', + 'storybook', + 'artifacts', + 'assessment-unit-ci', + assessmentDefinition.packagePath, + assessmentDefinition.component, + 'unit-execution.json', + ), + readFileSync(path.join(f.root, ...reportDirectory, 'unit-execution.json'), 'utf8'), + ); + f.env.NATIVE_OUTCOME = 'failure'; + const bundle = combineCiEvidence(f.root, f.env, now); + const report = assessCiBundle(f.root, bundle); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.requirements[0].outcome, 'partial'); + assert.equal(splitSpecAssessment(formatCiSpec(f.root, bundle)).assessment?.testedCommit, null); + put( + f.root, + path.join(...reportDirectory, 'bundle.json'), + JSON.stringify({ ...bundle, identity: { ...bundle.identity, runAttempt: 2 } }), + ); + assert.throws(() => combineCiEvidence(f.root, f.env, now), /different CI inputs/); + }); + + for (const mode of ['passed', 'failed'] as const) { + it(`reuses exactly one existing package invocation and retains its ${mode} exit status across jobs`, async () => { + const f = fixture(); + const command = path.join(repositoryRoot, 'scripts', 'src', 'ci', 'controlAssessmentCiTask.ts'); + const run = (argument: string) => + spawnSync(process.execPath, [command, argument], { + cwd: f.root, + env: { ...process.env, ...f.env }, + encoding: 'utf8', + }); + const begin = run('begin-unit'); + assert.equal(begin.status, 0, begin.stderr || begin.stdout); + const unitDirectory = path.join(f.root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'); + let invocations = 0; + const originalArgs = ['--runTestsByPath', 'checkbox.test.tsx', 'checkbox.types.test.ts']; + const unit = await runJestWithUnitCapture( + path.join(f.root, 'packages', 'agentic', 'components'), + 'macos', + originalArgs, + async (args) => { + invocations++; + assert.deepEqual(args.slice(0, originalArgs.length), originalArgs); + const data = componentJestFixture(); + const time = Date.now(); + data.startTime = time; + for (const suite of data.testResults) Object.assign(suite, { startTime: time, endTime: time }); + const outputIndex = args.indexOf('--outputFile'); + assert.ok(outputIndex >= 0, 'Existing invocation must receive its capture output path.'); + writeFileSync(args[outputIndex + 1], JSON.stringify(data)); + return mode === 'passed' ? 0 : 1; + }, + { ...f.env, FURN_ASSESSMENT_UNIT_CAPTURE: unitDirectory }, + ); + assert.equal(unit, mode === 'passed' ? 0 : 1); + assert.equal(invocations, 1); + // Another workspace may fail the aggregate without invalidating this successful component command. + f.env.UNIT_OUTCOME = 'failure'; + const nativeBegin = run('begin'); + assert.equal(nativeBegin.status, 0, nativeBegin.stderr || nativeBegin.stdout); + const time = new Date().toISOString(); + putNative(f.root, nativeReport({ startedAt: time, finishedAt: time })); + const finish = run('finish'); + assert.equal(finish.status, 0, finish.stderr || finish.stdout); + const combined = spawnSync( + process.execPath, + [ + path.join(repositoryRoot, 'scripts', 'src', 'cli.ts'), + 'assess-components', + '--package', + path.join(f.root, assessmentDefinition.packagePath), + '--ci', + ], + { cwd: f.root, env: { ...process.env, ...f.env }, encoding: 'utf8' }, + ); + assert.equal(combined.status, 0, combined.stderr || combined.stdout); + const report = JSON.parse(readFileSync(path.join(f.root, ...reportDirectory, 'assessment.json'), 'utf8')); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.packageSuites, 2); + assert.equal(report.unitEvidence.packageCounts.passed, 43); + assert.equal(report.unitEvidence.outcome, mode === 'passed' ? 'passed' : 'execution-failed'); + assert.equal(invocations, 1, 'Assessment must never invoke Jest again.'); + }); + } + + it('collects bound unit cases beside a native skip without claiming native qualification', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, f.env, now); + const report = assessCiBundle(f.root, bundle); + assert.equal(bundle.schemaVersion, 2); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.requirements[1].outcome, 'partial'); + assert.equal(report.requirements[5].outcome, 'insufficient-evidence'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.equal(report.evidenceSupportedStatus, null); + const matrix = report.requirements.find(({ id }) => id === 'CBX-004')!.checks; + assert.equal(matrix.find(({ id }) => id === 'js-checkbox-role')!.outcome, 'passed'); + assert.equal(matrix.find(({ id }) => id === 'native-checkbox-role')!.outcome, 'skipped'); + assert.equal(matrix.find(({ id }) => id === 'native-toggle-dispatch')!.outcome, 'uncovered'); + assert.match(formatCiSummary(f.root, bundle), /Recorded unit cases: 42 passed/); + assert.equal(splitSpecAssessment(formatCiSpec(f.root, bundle)).assessment?.testedCommit, f.testedCommit); + const context = bundle.unit!.context!; + for (const change of [ + { sourceCommit: f.headCommit }, + { reportSha256: 'f'.repeat(64) }, + { buildId: 'example/furn/actions/runs/456/attempts/2' }, + { finishedAt: '2026-10-22T20:00:00.000Z' }, + { workingTreeDirty: true }, + ]) { + assert.throws( + () => assessCiBundle(f.root, { ...bundle, unit: { ...bundle.unit!, context: { ...context, ...change } } }), + /binding|dirty/, + ); + } + }); + + it('preserves unit step failure instead of accepting passing JSON from a failed parent command', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + f.env.UNIT_OUTCOME = 'failure'; + const report = assessCiBundle(f.root, finishCiExecution(f.root, f.env, now)); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.equal(report.requirements[1].outcome, 'insufficient-evidence'); + }); + + it('keeps absent/skipped unit collection unavailable and accepts old native-only bundles explicitly', () => { + const f = fixture(); + f.env.UNIT_OUTCOME = 'skipped'; + const bundle = execute(f); + const report = assessCiBundle(f.root, bundle); + assert.equal(report.unitEvidence.status, 'missing'); + assert.match(report.unitEvidence.executionFailure ?? '', /skipped/); + const { unit, ...nativeOnly } = bundle; + assert.ok(unit); + const legacy = parseBundle(JSON.stringify({ ...nativeOnly, schemaVersion: 1 })); + assert.equal(legacy.unit, undefined); + assert.equal(assessCiBundle(f.root, legacy).unitEvidence.status, 'missing'); + assert.throws(() => parseBundle(JSON.stringify({ ...bundle, schemaVersion: 1 })), /Legacy/); + assert.throws( + () => parseBundle(JSON.stringify({ ...bundle, unit: { report: '{}', context: null, executionFailure: null } })), + /together/, + ); + }); + + it('rejects stale unit captures from a previous interval or workflow attempt', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + putUnit(f, (capture) => { + (capture.context as Record).startedAt = '2026-10-20T20:00:00.000Z'; + }); + assert.throws(() => finishCiExecution(f.root, f.env, now), /predates/); + putUnit(f, (capture) => { + (capture.identity as Record).runAttempt = 2; + }); + assert.throws(() => finishCiExecution(f.root, f.env, now), /different CI/); + }); + + it('captures before execution, retains an honest hosted skip, and writes usable reports', () => { + const f = fixture(); + const bundle = execute(f); + const report = assessCiBundle(f.root, bundle); + assert.equal(bundle.identity.testedCommit, f.testedCommit); + assert.equal(bundle.identity.headCommit, f.headCommit); + assert.equal(bundle.context?.physicalClick, false); + assert.equal(report.evidence.status, 'matched'); + assert.equal(report.requirements[1].check?.recordedOutcome, 'skipped'); + assert.equal(report.requirements[1].check?.evidenceOutcome, 'skipped'); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + const markdown = readFileSync(path.join(f.root, ...reportDirectory, 'ASSESSMENT.md'), 'utf8'); + assert.match(markdown, /AUTO-GENERATED\. DO NOT EDIT/); + assert.match(markdown, /physical pointer input/); + assert.equal(parseReportReference(markdown).inputDigest, inputDigest(f.root, f.testedCommit)); + assert.deepEqual(parseBundle(readFileSync(path.join(f.root, ...reportDirectory, 'bundle.json'), 'utf8')), bundle); + }); + + it('never binds an old report when pre-execution capture did not run', () => { + const f = fixture(); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, { ...f.env, SMOKE_OUTCOME: 'skipped' }, now); + assert.equal(bundle.nativeReport, null); + assert.equal(bundle.context, null); + assert.match(bundle.executionFailure ?? '', /skipped/); + assert.equal(assessCiBundle(f.root, bundle).evidence.status, 'missing'); + }); + + it('preserves but never binds desktop runs that existed before capture', () => { + const f = fixture(); + putNative(f.root, nativeReport()); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(bundle.nativeReport, null); + assert.match(bundle.executionFailure ?? '', /did not produce/); + assert.equal(JSON.parse(readFileSync(path.join(f.root, ...nativePath), 'utf8')).runId, 'synthetic-desktop-run'); + }); + + it('rejects multiple new desktop runs rather than choosing the newest report', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put(f.root, path.join(...desktopDirectory, 'another-run', 'run.json'), desktopEvidenceFixture(nativeReport()).canonical); + assert.throws(() => finishCiExecution(f.root, f.env, now), /Ambiguous/); + }); + + it('rejects a canonical run ID that differs from its captured artifact directory', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put(f.root, path.join(...nativePath), desktopEvidenceFixture(nativeReport(), { runId: 'different-run' }).canonical); + assert.throws(() => finishCiExecution(f.root, f.env, now), /artifact directory/); + }); + + it('rejects another generated target instance instead of accepting the Win32 prefix alone', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put( + f.root, + path.join(...desktopDirectory, 'synthetic-desktop-run', 'evidence', 'win32', 'static', 'run.json'), + nativeReport({ targetId: `win32-${'0'.repeat(16)}` }), + ); + assert.throws(() => finishCiExecution(f.root, f.env, now), /different native instance/); + }); + + for (const missing of ['undeclared companion', 'unwritten companion', 'unfinished canonical run'] as const) { + it(`reports unavailable evidence for ${missing}`, () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + if (missing === 'unfinished canonical run') { + put(f.root, path.join(...desktopDirectory, 'synthetic-desktop-run', 'events.jsonl'), '{}\n'); + } else { + put( + f.root, + path.join(...nativePath), + desktopEvidenceFixture(nativeReport(), missing === 'undeclared companion' ? { artifacts: [] } : {}).canonical, + ); + } + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(bundle.nativeReport, null); + assert.equal(bundle.context, null); + assert.match(bundle.executionFailure ?? '', /evidence|report/); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'insufficient-evidence'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /evidence|report/); + }); + } + + it('preserves a canonical cleanup failure in the generated assessment', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + put( + f.root, + path.join(...nativePath), + desktopEvidenceFixture(nativeReport(), { + status: 'failed', + failures: [{ failureId: 'cleanup', kind: 'cleanup', error: { name: 'Error', message: '' } }], + }).canonical, + ); + const bundle = finishCiExecution(f.root, f.env, now); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'execution-failed'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /Desktop supervisor status: failed/); + assert.match(formatCiSummary(f.root, bundle), /Desktop supervisor: failed/); + }); + + it('does not accept passes from a smoke step that failed after writing its report', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, { ...f.env, SMOKE_OUTCOME: 'failure' }, now); + assert.equal(assessCiBundle(f.root, bundle).assessmentOutcome, 'execution-failed'); + assert.match(assessCiBundle(f.root, bundle).evidence.runError ?? '', /failure/); + }); + + it('rejects dirty tracked and nonignored inputs instead of inventing a clean execution', () => { + const f = fixture(); + put(f.root, 'source.ts', 'dirty\n'); + assert.throws(() => beginCiExecution(f.root, f.env, now), /inputs changed/); + put(f.root, 'source.ts', 'export const value = 1;\n'); + put(f.root, 'extra.ts', 'untracked\n'); + assert.throws(() => finishCiExecution(f.root, f.env, now), /inputs changed/); + }); + + it('requires hosted capability disabling and the real merged checkout', () => { + const f = fixture(); + assert.throws(() => beginCiExecution(f.root, { ...f.env, FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES: '' }, now), /physicalClick/); + assert.throws(() => beginCiExecution(f.root, { ...f.env, GITHUB_SHA: f.headCommit }, now), /checkout/); + const bundle = execute(f); + assert.throws( + () => assessCiBundle(f.root, { ...bundle, identity: { ...bundle.identity, testedCommit: f.headCommit } }), + /merge commit/, + ); + }); + + it('rejects old report timestamps, changed source trees, and incorrect report hashes', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport({ startedAt: '2026-10-21T19:00:00.000Z' })); + assert.throws(() => finishCiExecution(f.root, f.env, now), /predates/); + const bundle = execute(f); + assert.throws(() => assessCiBundle(f.root, { ...bundle, identity: { ...bundle.identity, inputDigest: 'a'.repeat(64) } }), /digest/); + const context = bundle.context; + assert.ok(context); + assert.throws(() => assessCiBundle(f.root, { ...bundle, context: { ...context, reportSha256: 'a'.repeat(64) } }), /binding/); + }); + + it('rejects a physical pass on the explicitly noninteractive hosted runner', () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putNative( + f.root, + nativeReport({ + wdio: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'passed', + durationMs: 10, + }, + ], + }), + ); + assert.throws(() => finishCiExecution(f.root, f.env, now), /physical-click/); + }); + + it('runs the actual CI entry point and persists files without dependencies or network writes', () => { + const f = fixture(); + const command = path.join(repositoryRoot, 'scripts', 'src', 'ci', 'controlAssessmentCiTask.ts'); + const run = (argument: string) => + spawnSync(process.execPath, [command, argument], { + cwd: f.root, + env: { ...process.env, ...f.env }, + encoding: 'utf8', + }); + const begin = run('begin'); + assert.equal(begin.status, 0, begin.stderr || begin.stdout); + const time = new Date().toISOString(); + putNative(f.root, nativeReport({ startedAt: time, finishedAt: time })); + const finish = run('finish'); + assert.equal(finish.status, 0, finish.stderr || finish.stdout); + assert.match(finish.stdout, /No maturity stage assigned/); + assert.equal(parseBundle(readFileSync(path.join(f.root, ...reportDirectory, 'bundle.json'), 'utf8')).identity.headCommit, f.headCommit); + }); +}); + +describe('same-PR source fingerprint and evidence reuse', () => { + it('verifies unit evidence through the same exact two-file publication and follow-up path', async () => { + const f = fixture(); + beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z')); + putUnit(f); + putNative(f.root, nativeReport()); + const bundle = finishCiExecution(f.root, f.env, now); + const summary = formatCiSummary(f.root, bundle); + const update = reportCommit(f, summary); + const { github } = fakeApi(f); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.equal(candidate.reportOnly, true); + verifyReportUpdate(f.root, candidate, bundle, now); + assert.ok(candidate.reportOnly); + assert.throws( + () => verifyReportUpdate(f.root, { ...candidate, mapping: `${candidate.mapping}\n` }, bundle, now), + /exact reviewed mapping/, + ); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const tampered = { + ...bundle, + unit: { ...bundle.unit!, report: bundle.unit!.report!.replace('"numPassedTests":42', '"numPassedTests":41') }, + }; + assert.throws(() => verifyReportUpdate(f.root, candidate, tampered, now), /counts|binding/); + }); + + it('preserves the legacy raw tree digest before generated SPEC metadata exists', () => { + const f = fixture(); + const raw = git(f.root, ['ls-tree', '-rz', '--full-tree', f.testedCommit]); + assert.equal(inputDigest(f.root, f.testedCommit), createHash('sha256').update(raw).digest('hex')); + }); + + it('verifies the report-only follow-up without relabeling its commit as tested', async () => { + const f = fixture(); + const bundle = execute(f); + const summary = formatCiSummary(f.root, bundle); + const update = reportCommit(f, summary); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, parseReportReference(summary), update.head, update.merged), true); + const { github } = fakeApi(f); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.equal(candidate.reportOnly, true); + verifyReportUpdate(f.root, candidate, bundle, now); + assert.notEqual(bundle.identity.testedCommit, update.merged); + assert.notEqual(bundle.identity.headCommit, update.head); + }); + + for (const file of [ + 'source.ts', + 'yarn.lock', + '.github/workflows/pr.yml', + `${contractPath}/SPEC.md`, + `${contractPath}/checkbox.test.tsx`, + `${contractPath}/assessment/not-a-generated-report.ts`, + assessmentPackagePath, + ]) { + it(`does not exclude source input ${file}`, () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put( + f.root, + file, + file === assessmentPackagePath + ? JSON.stringify({ ...JSON.parse(readFileIfPresent(f.root, file)), assessmentTestChange: true }) + : `${readFileIfPresent(f.root, file)}\n// changed input\n`, + ); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const changed = commit(f.root, 'Synthetic source and report changes'); + const merged = merge(f.root, f.baseCommit, changed); + assert.notEqual(inputDigest(f.root, merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, changed, merged), false); + }); + } + + it('records the tested package version and merge SHA, never the publication SHA', () => { + const f = fixture(); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle); + const metadata = splitSpecAssessment(spec).assessment; + assert.equal(metadata?.packageVersion, '0.7.0'); + assert.equal(metadata?.sourceCommit, f.testedCommit); + assert.equal(metadata?.testedCommit, f.testedCommit); + assert.equal(metadata?.dirty, false); + assert.equal(metadata?.provenance, 'matched'); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + assert.notEqual(metadata?.testedCommit, update.head); + assert.notEqual(metadata?.testedCommit, update.merged); + }); + + it('rejects altered generated SPEC fields even though their structural input digest is unchanged', async () => { + const f = fixture(); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle).replace("assessment-package-version: '0.7.0'", "assessment-package-version: '99.0.0'"); + const update = reportCommit(f, formatCiSummary(f.root, bundle), spec); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + assert.ok(candidate.reportOnly); + assert.throws(() => verifyReportUpdate(f.root, candidate, bundle, now), /SPEC.*trusted rendering/); + }); + + it('falls back to full CI for an old summary-only publication without a SPEC update', async () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const head = commit(f.root, 'Legacy summary-only update'); + f.env.GITHUB_SHA = merge(f.root, f.baseCommit, head); + writeFileSync(f.event, JSON.stringify(f.payload(head))); + assert.equal((await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now)).reportOnly, false); + }); + + it('does not normalize away executable mode or human contract changes in a two-file publication', () => { + for (const mode of [false, true]) { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + put(f.root, assessmentSpecPath, `${formatCiSpec(f.root, bundle)}${mode ? '' : '\nHuman contract changed.\n'}`); + git(f.root, ['add', '.']); + if (mode) { + chmodSync(path.join(f.root, assessmentSpecPath), 0o755); + git(f.root, ['update-index', '--chmod=+x', assessmentSpecPath]); + } + git(f.root, ['commit', '-qm', 'Changed actual SPEC inputs']); + const head = git(f.root, ['rev-parse', 'HEAD']); + if (mode) assert.match(git(f.root, ['ls-tree', head, '--', assessmentSpecPath]), /^100755 blob /); + const merged = merge(f.root, f.baseCommit, head); + assert.notEqual(inputDigest(f.root, merged), bundle.identity.inputDigest); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, head, merged), false); + } + }); + + it('renders from original PR head contract bytes but takes the version and assessment from the tested merge', async () => { + const f = fixture(); + const headSpec = readFileSync(path.join(f.root, assessmentSpecPath), 'utf8'); + git(f.root, ['checkout', '-q', 'main']); + put(f.root, assessmentSpecPath, `${headSpec}\nBase-only contract note.\n`); + put( + f.root, + assessmentPackagePath, + JSON.stringify({ ...JSON.parse(readFileSync(path.join(f.root, assessmentPackagePath), 'utf8')), version: '0.8.0' }), + ); + f.baseCommit = commit(f.root, 'New main contract and version'); + f.testedCommit = merge(f.root, f.baseCommit, f.headCommit); + f.env.GITHUB_SHA = f.testedCommit; + writeFileSync(f.event, JSON.stringify(f.payload(f.headCommit, f.baseCommit))); + const bundle = execute(f); + const spec = formatCiSpec(f.root, bundle); + assert.equal(splitSpecAssessment(spec).contract, headSpec); + assert.equal(splitSpecAssessment(spec).assessment?.packageVersion, '0.8.0'); + assert.equal(splitSpecAssessment(spec).assessment?.health, 'stale'); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + assert.equal(inputDigest(f.root, update.merged), bundle.identity.inputDigest); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + verifyReportUpdate(f.root, candidate, bundle, now); + }); + + it('rejects non-UTF-8 SPEC bytes instead of folding distinct source bytes together', () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + writeFileSync(path.join(f.root, assessmentSpecPath), Buffer.concat([Buffer.from(formatCiSpec(f.root, bundle)), Buffer.from([0xff])])); + const head = commit(f.root, 'Invalid SPEC encoding'); + assert.throws(() => inputDigest(f.root, head), /valid UTF-8/); + }); + + it('detects changes in earlier commits, not just the last report commit', () => { + const f = fixture(); + const bundle = execute(f); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + put(f.root, 'source.ts', 'changed after testing\n'); + commit(f.root, 'Synthetic earlier code commit'); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, bundle)); + const head = commit(f.root, 'Synthetic latest report-only commit'); + const merged = merge(f.root, f.baseCommit, head); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, head, merged), false); + }); + + it('invalidates reuse when another control PR changes the base branch', () => { + const f = fixture(); + const bundle = execute(f); + const update = reportCommit(f, formatCiSummary(f.root, bundle)); + git(f.root, ['checkout', '-q', 'main']); + put(f.root, 'other-control.ts', 'another merged change\n'); + const newBase = commit(f.root, 'Synthetic concurrent PR'); + const merged = merge(f.root, newBase, update.head); + assert.equal(isReportOnlyUpdate(f.root, { ...bundle.identity, generatedAt: bundle.generatedAt }, update.head, merged), false); + }); + + it('recovers from concurrent generated-report conflicts by resolving source and reassessing the combined tree', () => { + const f = fixture(); + const firstBundle = execute(f); + const first = reportCommit(f, formatCiSummary(f.root, firstBundle)); + git(f.root, ['checkout', '-q', 'main']); + git(f.root, ['merge', '--ff-only', first.head]); + git(f.root, ['checkout', '-qb', 'pilot/second', f.baseCommit]); + put(f.root, 'other-control.ts', 'second PR source\n'); + const secondHead = commit(f.root, 'Synthetic simultaneous control PR'); + const secondMerge = merge(f.root, f.baseCommit, secondHead); + const reassess = (head: string, base: string, tested: string) => { + const event = { + pull_request: { + number: 124, + head: { sha: head, ref: 'pilot/second', repo: { full_name: 'example/furn' } }, + base: { sha: base, ref: 'main', repo: { full_name: 'example/furn' } }, + }, + }; + writeFileSync(f.event, JSON.stringify(event)); + const env = { ...f.env, GITHUB_RUN_ID: '789', GITHUB_SHA: tested }; + rmSync(path.join(f.root, ...desktopDirectory, 'synthetic-desktop-run'), { recursive: true, force: true }); + beginCiExecution(f.root, env, new Date('2026-10-21T20:00:00.000Z')); + putNative(f.root, nativeReport()); + return finishCiExecution(f.root, env, now); + }; + const secondBundle = reassess(secondHead, f.baseCommit, secondMerge); + git(f.root, ['checkout', '-q', 'pilot/second']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, secondBundle)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, secondBundle)); + commit(f.root, 'Synthetic second generated report'); + const conflict = spawnSync('git', ['merge', '--no-ff', '--no-edit', 'main'], { cwd: f.root, encoding: 'utf8' }); + assert.equal(conflict.status, 1, conflict.stdout || conflict.stderr); + assert.deepEqual( + git(f.root, ['diff', '--name-only', '--diff-filter=U']).split('\n').sort(), + [assessmentSummaryPath, assessmentSpecPath].sort(), + ); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, firstBundle)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, firstBundle)); + const updatedHead = commit(f.root, 'Resolve generated report with main snapshot temporarily'); + const updatedMerge = merge(f.root, first.head, updatedHead); + const refreshed = reassess(updatedHead, first.head, updatedMerge); + assert.notEqual(refreshed.identity.inputDigest, secondBundle.identity.inputDigest); + assert.equal(refreshed.identity.baseCommit, first.head); + git(f.root, ['checkout', '-q', 'pilot/second']); + put(f.root, assessmentSummaryPath, formatCiSummary(f.root, refreshed)); + put(f.root, assessmentSpecPath, formatCiSpec(f.root, refreshed)); + const reportHead = commit(f.root, 'Regenerate second PR assessment after integration'); + const reportMerge = merge(f.root, first.head, reportHead); + assert.equal(isReportOnlyUpdate(f.root, { ...refreshed.identity, generatedAt: refreshed.generatedAt }, reportHead, reportMerge), true); + }); + + it('rejects a manually altered report body even when the embedded source reference is valid', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, `${formatCiSummary(f.root, bundle)}\nManual readiness claim\n`); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + assert.equal(candidate.reportOnly, true); + assert.throws(() => verifyReportUpdate(f.root, candidate, bundle, now), /trusted rendering/); + }); + + it('preserves failed source checks rather than blessing them through skipped jobs', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + const { github } = fakeApi(f, { + '/repos/example/furn/actions/runs/456/attempts/1/jobs?per_page=100&page=1': { + total_count: 1, + jobs: [{ name: 'PR source validation', conclusion: 'failure' }], + }, + }); + const candidate = await inspectReportUpdate(f.root, f.env, github, now); + assert.ok(candidate.reportOnly); + assert.equal(candidate.sourceResult, 'failure'); + verifyReportUpdate(f.root, candidate, bundle, now); + }); + + it('fails closed for a missing artifact or a source attempt with the wrong head', async () => { + const f = fixture(); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + await assert.rejects( + inspectReportUpdate( + f.root, + f.env, + fakeApi(f, { + '/repos/example/furn/actions/runs/456/artifacts?per_page=100&page=1': { total_count: 0, artifacts: [] }, + }).github, + now, + ), + /missing or expired/, + ); + await assert.rejects( + inspectReportUpdate( + f.root, + f.env, + fakeApi(f, { + '/repos/example/furn/actions/runs/456/attempts/1': { + id: 456, + run_attempt: 1, + event: 'pull_request', + status: 'completed', + path: '.github/workflows/pr.yml', + head_sha: 'a'.repeat(40), + head_branch: 'pilot/checkbox', + head_repository: { full_name: 'example/furn' }, + }, + }).github, + now, + ), + /source PR workflow/, + ); + }); + + it('runs full CI when there is no reusable report or its explicit reuse age has expired', async () => { + const f = fixture(); + assert.equal((await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now)).reportOnly, false); + const bundle = execute(f); + reportCommit(f, formatCiSummary(f.root, bundle)); + const later = new Date(now.getTime() + 169 * 3_600_000); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, later); + assert.equal(candidate.reportOnly, false); + assert.throws(() => assertFresh(bundle.generatedAt, later), /reuse limit/); + }); +}); + +describe('trusted report publication', () => { + it('publishes two configured controls atomically and verifies the complete report-only follow-up', async () => { + const f = fixture(true); + const definitions = discoverAssessments(f.root); + for (const definition of definitions) beginCiExecution(f.root, f.env, new Date('2026-10-21T20:00:00.000Z'), definition); + putNative(f.root, nativeReport()); + const bundles = definitions.map((definition) => finishCiExecution(f.root, f.env, now, definition)); + const expected = { repository: 'example/furn', runId: '456', runAttempt: 1, defaultBranch: 'main' }; + const { github, calls } = fakeApi(f); + await assert.rejects(preparePublications(f.root, bundles.slice(1), github, expected, now), /every configured component/); + const publications = await preparePublications(f.root, bundles, github, expected, now); + await publishAssessments(publications, github); + const mutations = calls.filter(({ url }) => url.endsWith('/graphql')); + assert.equal(mutations.length, 1); + const payload = JSON.parse(String(mutations[0].options?.body)); + assert.equal(payload.variables.input.expectedHeadOid, f.headCommit); + assert.deepEqual( + payload.variables.input.fileChanges.additions.map((addition: { path: string }) => addition.path).sort(), + definitions.flatMap(({ summaryPath, specPath, mappingPath }) => [summaryPath, specPath, mappingPath]).sort(), + ); + git(f.root, ['checkout', '-q', 'pilot/checkbox']); + for (const publication of publications) { + assert.ok(publication.eligible); + put(f.root, publication.summaryPath, publication.summary); + put(f.root, publication.specPath, publication.spec); + put(f.root, publication.mappingPath, publication.mapping); + } + const head = commit(f.root, 'Synthetic atomic component reports'); + const merged = merge(f.root, f.baseCommit, head); + writeFileSync(f.event, JSON.stringify(f.payload(head, f.baseCommit))); + f.env.GITHUB_SHA = merged; + const candidate = await inspectReportUpdates(f.root, f.env, fakeApi(f).github, now); + assert.ok(candidate.reportOnly); + assert.equal(candidate.candidates.length, 2); + verifyReportUpdates(f.root, candidate, bundles, now); + assert.throws(() => verifyReportUpdates(f.root, candidate, bundles.slice(1), now), /every configured component/); + assert.throws( + () => verifyReportUpdates(f.root, { ...candidate, candidates: [candidate.candidates[0], candidate.candidates[0]] }, bundles, now), + /every configured component/, + ); + }); + + const expected = { repository: 'example/furn', runId: '456', runAttempt: 1, defaultBranch: 'main' }; + + it('writes exactly the summary, SPEC and reviewed mapping with one atomic expected-head check', async () => { + const f = fixture(); + const bundle = execute(f); + const { github, calls } = fakeApi(f); + const publication = await preparePublication(f.root, bundle, github, expected, now); + assert.ok(publication.eligible); + assert.equal(calls.filter(({ options }) => options?.method === 'POST').length, 0); + const sha = await publishAssessment(publication, github); + assert.equal(sha, 'd'.repeat(40)); + const mutation = calls.find(({ url }) => url.endsWith('/graphql')); + assert.ok(mutation); + const payload = JSON.parse(String(mutation.options?.body)); + assert.equal(payload.variables.input.expectedHeadOid, f.headCommit); + assert.deepEqual(payload.variables.input.branch, { repositoryNameWithOwner: 'example/furn', branchName: 'pilot/checkbox' }); + assert.equal(payload.variables.input.fileChanges.additions.length, 3); + assert.equal(payload.variables.input.fileChanges.additions[0].path, assessmentSummaryPath); + assert.equal(payload.variables.input.fileChanges.additions[1].path, assessmentSpecPath); + assert.equal(Buffer.from(payload.variables.input.fileChanges.additions[1].contents, 'base64').toString(), formatCiSpec(f.root, bundle)); + assert.equal(payload.variables.input.fileChanges.additions[2].path, assessmentMappingPath); + assert.equal( + Buffer.from(payload.variables.input.fileChanges.additions[2].contents, 'base64').toString(), + formatCiMapping(f.root, bundle), + ); + assert.equal(payload.variables.input.fileChanges.deletions, undefined); + }); + + for (const [label, overrides] of [ + ['newer source commit', { head: { sha: 'a'.repeat(40), ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } } }], + ['closed PR', { state: 'closed' }], + ['fork PR', { head: { sha: 'a'.repeat(40), ref: 'pilot/checkbox', repo: { full_name: 'contributor/furn' } } }], + ['conflicted PR', { merge_commit_sha: null }], + ['protected main branch as source', { head: { sha: 'a'.repeat(40), ref: 'main', repo: { full_name: 'example/furn' } } }], + ] as const) { + it(`does not write for ${label}`, async () => { + const f = fixture(); + const bundle = execute(f); + const pr = { + number: 123, + state: 'open', + base: { sha: f.baseCommit, ref: 'main', repo: { full_name: 'example/furn' } }, + head: { sha: f.headCommit, ref: 'pilot/checkbox', repo: { full_name: 'example/furn' } }, + merge_commit_sha: f.testedCommit, + ...overrides, + }; + const { github, calls } = fakeApi(f, { '/repos/example/furn/pulls/123': pr }); + const publication = await preparePublication(f.root, bundle, github, expected, now); + assert.equal(publication.eligible, false); + assert.equal(calls.filter(({ options }) => options?.method === 'POST').length, 0); + }); + } + + it('rejects an artifact from a different source run before requesting a PR or creating a commit', async () => { + const f = fixture(); + const bundle = execute(f); + const { github, calls } = fakeApi(f); + await assert.rejects(preparePublication(f.root, bundle, github, { ...expected, runId: '999' }, now), /triggering workflow/); + assert.equal(calls.length, 0); + }); + + it('surfaces a branch race rejected by GitHub rather than reporting publication success', async () => { + const f = fixture(); + const bundle = execute(f); + const { github } = fakeApi(f, { '/graphql': { errors: [{ message: 'expectedHeadOid mismatch' }] } }); + const publication = await preparePublication(f.root, bundle, github, expected, now); + await assert.rejects(publishAssessment(publication, github), /rejected the assessment commit/); + }); + + it('surfaces permission errors without logging credentials or returning an empty success', async () => { + const fetchImplementation: typeof fetch = async () => new Response('{}', { status: 403 }); + const github = new AssessmentGitHub({ token: 'synthetic-token', fetchImplementation }); + await assert.rejects(github.request('/repos/example/furn/pulls/123'), (error: Error) => { + assert.match(error.message, /403/); + assert.doesNotMatch(error.message, /synthetic-token/); + return true; + }); + }); +}); + +describe('CI input schema and workflow wiring', () => { + it('accepts formatter-stable generated output but still rejects altered assessment findings', async () => { + const f = fixture(); + const bundle = execute(f); + const summary = formatCiSummary(f.root, bundle); + assert.equal(formatAssessmentDocument(summary), summary); + reportCommit(f, summary); + const candidate = await inspectReportUpdate(f.root, f.env, fakeApi(f).github, now); + assert.ok(candidate.reportOnly); + verifyReportUpdate(f.root, candidate, bundle, now); + assert.throws( + () => + verifyReportUpdate( + f.root, + { + ...candidate, + summary: candidate.summary.replace('Readiness: not established.', 'Readiness: approved.'), + }, + bundle, + now, + ), + /trusted rendering/, + ); + }); + + it('rejects invalid IDs, unsafe references, malformed markers, and oversized bundles', () => { + for (const value of [null, {}, { repository: '../repo' }]) assert.throws(() => parseIdentity(value)); + assert.throws(() => parseReportReference('# Manual report')); + assert.throws(() => parseReportReference('')); + assert.throws(() => parseBundle('x'.repeat(2_000_001)), /size limit/); + assert.throws(() => assertFresh('2026-10-22T20:00:00.000Z', now), /future/); + }); + + it('keeps required checks alive, captures before smoke, and runs publisher code only from trusted main', () => { + const workflow = readFileSync(path.join(repositoryRoot, '.github', 'workflows', 'pr.yml'), 'utf8'); + assert.match(workflow, /name: PR\non: pull_request/); + assert.doesNotMatch(workflow, /paths-ignore:/); + assert.match(workflow, / {2}pr:\n {4}name: PR\n {4}if: always\(\)/); + assert.match(workflow, /The original source validation did not pass/); + const win32 = workflow.slice(workflow.indexOf(' win32-storybook:'), workflow.indexOf(' check-changesets:')); + assert.ok(win32.indexOf('controlAssessmentCiTask.ts begin') < win32.indexOf('name: Smoke test Win32 Storybook')); + assert.ok(win32.indexOf('controlAssessmentCiTask.ts finish') > win32.indexOf('name: Smoke test Win32 Storybook')); + assert.match(win32, /disabled-input-features: keyboard,physicalClick,wheel/); + assert.match(win32, /component-native-evidence-\$\{\{ github.run_attempt \}\}/); + const js = workflow.slice(workflow.indexOf(' js:'), workflow.indexOf(' bundle-size:')); + assert.match(js, /FURN_ASSESSMENT_UNIT_CAPTURE:/); + assert.equal(js.match(/run: yarn lage buildci/g)?.length, 1); + assert.doesNotMatch(workflow, /--run-unit-tests|controlAssessmentCiTask\.ts unit(?:\s|$)/); + assert.match(workflow, /needs: \[assessment-inputs, js, win32-storybook\]/); + assert.match(workflow, /FURN_ASSESSMENT_CI: 'true'/); + assert.match(workflow, /run: node \.yarn\/releases\/yarn-4\.18\.0\.cjs lage assess-components/); + const jest = readFileSync(path.join(repositoryRoot, 'scripts', 'src', 'tasks', 'jest.ts'), 'utf8'); + assert.match(jest, /runJestWithUnitCapture\(\s*context.root/); + const publisher = readFileSync(path.join(repositoryRoot, '.github', 'workflows', 'pr-control-assessment.yml'), 'utf8'); + assert.match(publisher, /workflow_run:/); + assert.match(publisher, /ref: \$\{\{ github.event.repository.default_branch \}\}/); + assert.doesNotMatch(publisher, /pull_request_target|run: yarn|ref: \$\{\{ github.event.workflow_run.head_sha/); + assert.match(publisher, /npm install --prefix "\$RUNNER_TEMP\/assessment-tools" --ignore-scripts --no-audit --no-fund oxfmt@0\.61\.0/); + assert.ok( + publisher.indexOf('Validate publication without write credentials') < publisher.indexOf('Create narrowly scoped publisher token'), + ); + assert.match(publisher, /permission-contents: write/); + assert.doesNotMatch(publisher, /permission-workflows:|permission-administration:|skip-token-revoke:/); + }); +}); + +function readFileIfPresent(root: string, file: string): string { + try { + return readFileSync(path.join(root, file), 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return ''; + throw error; + } +} diff --git a/scripts/src/ci/assessment/controlAssessmentCi.ts b/scripts/src/ci/assessment/controlAssessmentCi.ts new file mode 100644 index 0000000000..3f176359d3 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentCi.ts @@ -0,0 +1,631 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +import { + assessControl, + formatAssessmentMarkdown, + hasStorybookArtifact, + hex, + object, + parseEvidenceContext, + parseNativeReport, + readControlContractFrom, + readNativeEvidence, + reportDigest, + text, + timestamp, + win32StorybookTargetId, +} from './controlAssessment.ts'; +import type { EvidenceContext } from './controlAssessment.ts'; +import { specAssessmentFor, splitSpecAssessment, updateSpecAssessment } from './controlAssessmentSpec.ts'; +import { parseUnitContext, readUnitMappingFrom } from './controlAssessmentUnit.ts'; +import type { UnitContext } from './controlAssessmentUnit.ts'; +import { assessmentDirectory, discoverAssessments, selectAssessment, unitCaptureDirectory } from './assessmentConfig.ts'; +import type { AssessmentDefinition } from './assessmentConfig.ts'; +import { formatAssessmentDocument } from './assessmentMarkdown.ts'; + +export const assessmentArtifactName = 'component-assessments'; +export const assessmentMaxAgeHours = 168; +export const assessmentWorkflowPath = '.github/workflows/pr.yml'; +const marker = '') || first.length > 4096) { + throw new Error('Missing or invalid generated assessment reference.'); + } + const encoded = first.slice(marker.length, -4); + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(encoded)) throw new Error('Invalid assessment reference encoding.'); + const data = object(JSON.parse(Buffer.from(encoded, 'base64').toString('utf8')), 'Report reference'); + return { ...parseIdentity(data), generatedAt: timestamp(data.generatedAt, 'Report generation time') }; +} + +export function assertFresh(generatedAt: string, now: Date): void { + const age = now.getTime() - Date.parse(timestamp(generatedAt, 'Assessment generation time')); + if (!Number.isFinite(age) || age < 0 || age > assessmentMaxAgeHours * 3_600_000) { + throw new Error(`Assessment is from the future or older than the explicit ${assessmentMaxAgeHours}-hour CI reuse limit.`); + } +} + +export function assertTestedInputs(root: string, identity: CiIdentity): void { + ensureCommit(root, identity.testedCommit); + const parents = git(root, ['rev-list', '--parents', '-n', '1', identity.testedCommit]).split(' '); + if (parents.length !== 3 || parents[1] !== identity.baseCommit || parents[2] !== identity.headCommit) { + throw new Error('CI evidence must identify the actual PR merge commit and both source parents.'); + } + if (inputDigest(root, identity.testedCommit) !== identity.inputDigest) throw new Error('CI input digest does not match the tested tree.'); +} + +export function assessCiBundle(root: string, bundle: CiBundle) { + const definition = assessmentForBundle(root, bundle); + assertTestedInputs(root, bundle.identity); + const read = (file: string) => { + const content = gitFile(root, bundle.identity.testedCommit, file); + if (content === undefined) throw new Error(`Tested contract file is missing: ${file}.`); + return content; + }; + const contract = readControlContractFrom(read, definition); + if (bundle.context && Date.parse(bundle.context.finishedAt) > Date.parse(bundle.generatedAt)) { + throw new Error('Native execution finished after its assessment was generated.'); + } + const expectedBuildId = `${bundle.identity.repository}/actions/runs/${bundle.identity.runId}/attempts/${bundle.identity.runAttempt}`; + const report = assessControl({ + contract, + sourceCommit: bundle.identity.testedCommit, + workingTreeDirty: false, + now: new Date(bundle.generatedAt), + expectedBuildId, + expectedTargetId: bundle.context?.targetId, + maxAgeHours: assessmentMaxAgeHours, + report: bundle.nativeReport === null ? undefined : parseNativeReport(JSON.parse(bundle.nativeReport)), + reportSha256: bundle.nativeReport === null ? undefined : reportDigest(bundle.nativeReport), + context: bundle.context ?? undefined, + executionFailure: bundle.executionFailure ?? undefined, + unit: bundle.unit + ? { + mapping: bundle.unit.report === null ? undefined : readUnitMappingFrom(read, definition), + report: bundle.unit.report ?? undefined, + context: bundle.unit.context ?? undefined, + executionFailure: bundle.unit.executionFailure ?? undefined, + expectedBuildId, + } + : undefined, + }); + if (report.evidence.status === 'mismatched' || report.evidence.status === 'stale') { + throw new Error(`CI evidence binding failed: ${report.evidence.reasons.join(' ')}`); + } + if (report.unitEvidence.status === 'mismatched' || report.unitEvidence.status === 'stale') { + throw new Error(`CI unit evidence binding failed: ${report.unitEvidence.reasons.join(' ')}`); + } + if (bundle.unit?.context?.workingTreeDirty) throw new Error('CI unit evidence was captured from dirty source inputs.'); + return report; +} + +export function formatCiSummary(root: string, bundle: CiBundle): string { + const definition = assessmentForBundle(root, bundle); + const report = assessCiBundle(root, bundle); + const reference: ReportReference = { ...bundle.identity, generatedAt: bundle.generatedAt }; + const encoded = Buffer.from(JSON.stringify(reference)).toString('base64'); + return formatAssessmentDocument( + [ + `${marker}${encoded} -->`, + '', + '', + ...formatAssessmentMarkdown(report).trimEnd().split('\n'), + '', + '## CI source and freshness', + '', + `Assessed at: ${bundle.generatedAt}.`, + `Pull request source: \`${bundle.identity.headCommit}\`. Tested base: \`${bundle.identity.baseCommit}\`.`, + `All tracked input digest: \`${bundle.identity.inputDigest}\`.`, + `Execution: https://github.com/${bundle.identity.repository}/actions/runs/${bundle.identity.runId}/attempts/${bundle.identity.runAttempt}`, + '', + 'This is a snapshot of that execution, not a claim that the latest revision has passed.', + 'Only configured generated summaries and exact validated SPEC assessment blocks are excluded from the input digest. All other contract bytes, mappings, code, tests, dependencies, workflows, and configuration remain inputs.', + 'A report-only commit is not relabeled as a tested commit. CI may reuse the original results only after checking unchanged inputs and the original required PR result.', + ...(bundle.executionFailure + ? ['', `CI execution limitation: ${bundle.executionFailure.replace(/[<>]/g, '').replace(/[\r\n]/g, ' ')}`] + : []), + '', + ].join('\n'), + definition.summaryPath, + ); +} + +export function formatCiSpec(root: string, bundle: CiBundle): string { + const definition = assessmentForBundle(root, bundle); + const assessmentSpecPath = definition.specPath; + const assessmentPackagePath = `${definition.packagePath}/package.json`; + const report = assessCiBundle(root, bundle); + for (const commit of [bundle.identity.headCommit, bundle.identity.testedCommit]) { + if (!git(root, ['ls-tree', commit, '--', assessmentSpecPath]).startsWith('100644 blob ')) { + throw new Error('Assessment publication requires a regular nonexecutable SPEC file.'); + } + } + const manifest = gitFile(root, bundle.identity.testedCommit, assessmentPackagePath); + const spec = gitFile(root, bundle.identity.headCommit, assessmentSpecPath); + if (manifest === undefined || spec === undefined) throw new Error('Tested component manifest or source SPEC is missing.'); + const version = text(object(JSON.parse(manifest), 'Tested component manifest').version, 'Tested component version'); + return updateSpecAssessment(spec, specAssessmentFor(report, version)); +} + +export function formatCiMapping(root: string, bundle: CiBundle): string { + const { mappingPath } = assessmentForBundle(root, bundle); + for (const commit of [bundle.identity.headCommit, bundle.identity.testedCommit]) { + if (!git(root, ['ls-tree', commit, '--', mappingPath]).startsWith('100644 blob ')) + throw new Error('Assessment mapping must be a regular nonexecutable file.'); + } + const mapping = gitFile(root, bundle.identity.testedCommit, mappingPath); + if (mapping === undefined || gitFile(root, bundle.identity.headCommit, mappingPath) !== mapping) + throw new Error('Tested mapping differs from the PR source mapping; review and reconcile it before publication.'); + return mapping; +} + +function identityFromEnvironment(root: string, env: NodeJS.ProcessEnv): CiIdentity { + const eventPath = text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'); + const event = object(JSON.parse(readFileSync(eventPath, 'utf8')), 'PR event'); + const pr = object(event.pull_request, 'PR event pull_request'); + const head = object(pr.head, 'PR head'); + const base = object(pr.base, 'PR base'); + const testedCommit = hex(env.GITHUB_SHA, 40, 'GITHUB_SHA'); + const identity = parseIdentity({ + repository: env.GITHUB_REPOSITORY, + pullRequest: pr.number, + headRepository: object(head.repo, 'PR head repository').full_name, + headRef: head.ref, + headCommit: head.sha, + baseCommit: base.sha, + testedCommit, + runId: env.GITHUB_RUN_ID, + runAttempt: Number(env.GITHUB_RUN_ATTEMPT), + inputDigest: inputDigest(root, testedCommit), + }); + if (git(root, ['rev-parse', 'HEAD']) !== testedCommit) throw new Error('CI checkout does not match GITHUB_SHA.'); + assertTestedInputs(root, identity); + return identity; +} + +function assertCleanInputs(root: string): void { + if ( + git(root, [ + 'status', + '--porcelain', + '--untracked-files=all', + '--', + '.', + ...discoverAssessments(root).map(({ summaryPath }) => `:(exclude)${summaryPath}`), + ]) + ) { + throw new Error('CI tracked or nonignored inputs changed outside the generated assessment summary.'); + } +} + +export function beginCiExecution(root: string, env: NodeJS.ProcessEnv, now = new Date(), definition = selectAssessment(root)): void { + const directory = assessmentDirectory(root, definition); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + if ( + !(env.FURN_DESKTOP_DRIVER_DISABLED_INPUT_FEATURES ?? '') + .split(',') + .map((value) => value.trim()) + .includes('physicalClick') + ) { + throw new Error('The hosted pilot requires explicit physicalClick disabling; do not claim interactive-runner qualification.'); + } + const contract = readControlContractFrom((file) => readFileSync(path.join(root, file), 'utf8'), definition); + mkdirSync(directory, { recursive: true }); + const start: ExecutionStart = { + schemaVersion: 2, + identity, + contractDigest: contract.digest, + startedAt: now.toISOString(), + existingRuns: desktopRuns(root), + targetId: win32StorybookTargetId(root), + }; + writeFileSync(path.join(directory, 'execution-start.json'), `${JSON.stringify(start, null, 2)}\n`); +} + +export function beginCiUnitCollection(root: string, env: NodeJS.ProcessEnv, now = new Date()): void { + const directory = path.join(root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + mkdirSync(directory, { recursive: true }); + if ( + discoverAssessments(root).some((definition) => + existsSync(path.join(unitCaptureDirectory(directory, definition), 'unit-execution.json')), + ) + ) + throw new Error('Unit results already exist before CI collection.'); + writeFileSync( + path.join(directory, 'unit-collection-start.json'), + `${JSON.stringify( + { + schemaVersion: 1, + identity, + startedAt: now.toISOString(), + buildId: `${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`, + }, + null, + 2, + )}\n`, + ); +} + +export function finishCiExecution(root: string, env: NodeJS.ProcessEnv, now = new Date(), definition = selectAssessment(root)): CiBundle { + const directory = assessmentDirectory(root, definition); + mkdirSync(directory, { recursive: true }); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + const startPath = path.join(directory, 'execution-start.json'); + const started = existsSync(startPath) ? object(JSON.parse(readFileSync(startPath, 'utf8')), 'Execution start') : undefined; + if (started && started.schemaVersion !== 2) throw new Error('Unsupported pre-execution capture schema.'); + if (started && JSON.stringify(parseIdentity(started.identity)) !== JSON.stringify(identity)) { + throw new Error('CI inputs or identity changed during native execution.'); + } + if (started && (!Array.isArray(started.existingRuns) || started.existingRuns.some((name) => typeof name !== 'string'))) { + throw new Error('Invalid pre-execution desktop run inventory.'); + } + const freshRuns = started ? desktopRuns(root).filter((name) => !(started.existingRuns as string[]).includes(name)) : []; + if (freshRuns.length > 1) throw new Error('Ambiguous native evidence: more than one desktop run started during smoke.'); + const reportPath = freshRuns.length ? path.join(root, ...desktopRunsPath, freshRuns[0], 'run.json') : undefined; + let nativeReport: string | null = null; + let collectionFailure: string | undefined; + if (reportPath && existsSync(reportPath)) { + const canonical = object(JSON.parse(readFileSync(reportPath, 'utf8')), 'Desktop supervisor report'); + if (hasStorybookArtifact(canonical)) { + if (existsSync(path.join(path.dirname(reportPath), 'evidence', 'win32', 'static', 'run.json'))) { + nativeReport = readNativeEvidence(reportPath); + } else { + collectionFailure = 'Desktop supervisor declared Win32 Storybook smoke evidence, but the companion report is missing.'; + } + } else { + collectionFailure = 'Desktop supervisor produced no Win32 Storybook smoke evidence; native assessment is unavailable.'; + } + } + let context: EvidenceContext | null = null; + if (nativeReport !== null) { + const parsed = parseNativeReport(JSON.parse(nativeReport)); + if (parsed.format !== 'desktop-test' || !parsed.finishedAt || !parsed.startedAt) { + throw new Error('CI requires the canonical desktop-test result and its timestamped smoke evidence.'); + } + const began = timestamp(started?.startedAt, 'Execution start time'); + if (Date.parse(parsed.startedAt) < Date.parse(began)) { + throw new Error('Desktop report predates this execution.'); + } + if (parsed.harness?.runId !== freshRuns[0]) throw new Error('Desktop run ID does not match its artifact directory.'); + if (parsed.targetId !== text(started?.targetId, 'Execution target ID')) + throw new Error('Desktop report targets a different native instance from the pre-execution capture.'); + context = parseEvidenceContext({ + schemaVersion: 1, + reportSha256: reportDigest(nativeReport), + sourceCommit: identity.testedCommit, + buildId: `${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`, + contractDigest: hex(started?.contractDigest, 64, 'Execution contract digest'), + platformManifestDigest: parsed.platformManifestDigest, + endpoint: 'win32', + renderer: 'paper', + targetId: text(started?.targetId, 'Execution target ID'), + finishedAt: parsed.finishedAt, + physicalClick: false, + }); + } + const outcome = text(env.SMOKE_OUTCOME, 'SMOKE_OUTCOME'); + if (!['success', 'failure', 'cancelled', 'skipped'].includes(outcome)) throw new Error('Unsupported smoke step outcome.'); + const executionFailure = + outcome !== 'success' + ? `Win32 smoke step outcome: ${outcome}. Recorded results do not establish successful native execution.` + : nativeReport === null + ? (collectionFailure ?? 'Win32 smoke did not produce a current native report.') + : null; + const unit = readCiUnitCapture(directory, identity, env.UNIT_OUTCOME); + const bundle = parseBundle( + JSON.stringify({ + schemaVersion: 2, + component: definition.component, + packagePath: definition.packagePath, + identity, + generatedAt: now.toISOString(), + nativeReport, + context, + executionFailure, + unit, + }), + ); + writeCiAssessment(root, directory, bundle, env); + return bundle; +} + +function readCiUnitCapture(directory: string, identity: CiIdentity, unitOutcome?: string): NonNullable { + const unitPath = path.join(directory, 'unit-execution.json'); + let unit: NonNullable = { report: null, context: null, executionFailure: null }; + if (unitOutcome !== undefined && !['success', 'failure', 'cancelled', 'skipped'].includes(unitOutcome)) { + throw new Error('Unsupported unit step outcome.'); + } + if (existsSync(unitPath)) { + const captured = object(JSON.parse(readFileSync(unitPath, 'utf8')), 'Unit execution capture'); + if (captured.schemaVersion !== 1 || JSON.stringify(parseIdentity(captured.identity)) !== JSON.stringify(identity)) { + throw new Error('Unit execution capture belongs to different CI inputs or attempt.'); + } + const context = captured.context === null ? null : parseUnitContext(captured.context); + if (!Number.isSafeInteger(captured.exitCode) || Number(captured.exitCode) < 0) throw new Error('Invalid unit capture exit code.'); + if (context && Date.parse(context.startedAt) < Date.parse(timestamp(captured.collectionStartedAt, 'Unit collection start time'))) { + throw new Error('Unit report predates this CI execution.'); + } + if (captured.report !== null && typeof captured.report !== 'string') throw new Error('Invalid captured unit report.'); + if (captured.executionFailure !== null && typeof captured.executionFailure !== 'string') + throw new Error('Invalid captured unit failure.'); + unit = { + report: captured.report, + context, + executionFailure: + unitOutcome !== undefined && unitOutcome !== 'success' + ? `Component unit step outcome: ${unitOutcome}.` + : Number(captured.exitCode) !== 0 + ? `Component unit collector failed with exit ${captured.exitCode}.` + : captured.executionFailure, + }; + } else if (unitOutcome !== undefined) { + unit.executionFailure = `Component unit step outcome: ${unitOutcome}; no current execution capture was produced.`; + } + return unit; +} + +export function combineCiEvidence(root: string, env: NodeJS.ProcessEnv, now = new Date(), definition = selectAssessment(root)): CiBundle { + const directory = assessmentDirectory(root, definition); + mkdirSync(directory, { recursive: true }); + const identity = identityFromEnvironment(root, env); + assertCleanInputs(root); + const nativePath = path.join(directory, 'bundle.json'); + const native = existsSync(nativePath) ? parseBundle(readFileSync(nativePath, 'utf8')) : undefined; + if (native && (native.component !== definition.component || native.packagePath !== definition.packagePath)) + throw new Error('Native artifact belongs to a different configured component.'); + if (native && JSON.stringify(native.identity) !== JSON.stringify(identity)) + throw new Error('Native artifact belongs to different CI inputs or attempt.'); + if (native) assertFresh(native.generatedAt, now); + const unitDirectory = unitCaptureDirectory(path.join(root, 'apps', 'storybook', 'artifacts', 'assessment-unit-ci'), definition); + const unit = readCiUnitCapture( + unitDirectory, + identity, + existsSync(path.join(unitDirectory, 'unit-execution.json')) ? undefined : env.UNIT_OUTCOME, + ); + const bundle = parseBundle( + JSON.stringify({ + schemaVersion: 2, + component: definition.component, + packagePath: definition.packagePath, + identity, + generatedAt: now.toISOString(), + nativeReport: native?.nativeReport ?? null, + context: native?.context ?? null, + executionFailure: + native?.executionFailure ?? + (native ? null : `Native job outcome: ${text(env.NATIVE_OUTCOME, 'NATIVE_OUTCOME')}; no native bundle was collected.`), + unit, + }), + ); + writeCiAssessment(root, directory, bundle, env); + return bundle; +} + +function writeCiAssessment(root: string, directory: string, bundle: CiBundle, env: NodeJS.ProcessEnv): void { + const mapping = formatCiMapping(root, bundle); + writeFileSync(path.join(directory, 'assessment.mapping.json'), mapping); + const report = assessCiBundle(root, bundle); + const summary = formatCiSummary(root, bundle); + writeFileSync(path.join(directory, 'bundle.json'), `${JSON.stringify(bundle, null, 2)}\n`); + writeFileSync(path.join(directory, 'assessment.json'), `${JSON.stringify(report, null, 2)}\n`); + writeFileSync(path.join(directory, 'ASSESSMENT.md'), summary); + writeFileSync(path.join(directory, 'SPEC.md'), formatCiSpec(root, bundle)); + if (bundle.unit?.report !== null && bundle.unit?.report !== undefined) { + writeFileSync(path.join(directory, 'jest.json'), bundle.unit.report); + writeFileSync(path.join(directory, 'unit-context.json'), `${JSON.stringify(bundle.unit.context, null, 2)}\n`); + } + if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, summary); +} + +export function isReportOnlyUpdate(root: string, reference: ReportReference, head: string, merged: string): boolean { + const definitions = discoverAssessments(root); + const allowedPaths = definitions.flatMap(({ summaryPath, specPath }) => [summaryPath, specPath]); + ensureCommit(root, reference.headCommit); + ensureCommit(root, head); + ensureCommit(root, merged); + const parents = git(root, ['rev-list', '--parents', '-n', '1', head]).split(' '); + if (parents.length !== 2 || parents[1] !== reference.headCommit) return false; + const changed = git(root, ['diff', '--name-only', reference.headCommit, head]).split('\n'); + if (!definitions.length || changed.length !== allowedPaths.length || changed.some((file) => !allowedPaths.includes(file))) return false; + for (const file of allowedPaths) { + if (!git(root, ['ls-tree', merged, '--', file]).startsWith('100644 blob ')) return false; + } + + for (const definition of definitions) { + const spec = gitFile(root, head, definition.specPath); + if (spec === undefined || splitSpecAssessment(spec).assessment === undefined) return false; + } + assertTestedInputs(root, reference); + return inputDigest(root, merged) === reference.inputDigest; +} + +export function assessmentForBundle(root: string, bundle: CiBundle): AssessmentDefinition { + const packageRoot = bundle.packagePath === undefined ? undefined : path.resolve(root, bundle.packagePath); + return selectAssessment(root, bundle.component, packageRoot); +} diff --git a/scripts/src/ci/assessment/controlAssessmentFixtures.ts b/scripts/src/ci/assessment/controlAssessmentFixtures.ts new file mode 100644 index 0000000000..8695c56f7b --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentFixtures.ts @@ -0,0 +1,155 @@ +import { fileURLToPath } from 'node:url'; +import { reportDigest, readControlContractFrom as readContract } from './controlAssessment.ts'; +import { parseComponentJest as parseJest, readUnitMappingFrom as readMapping } from './controlAssessmentUnit.ts'; +import { selectAssessment } from './assessmentConfig.ts'; + +export const assessmentDefinition = selectAssessment(fileURLToPath(new URL('../../../../', import.meta.url)), 'checkbox'); +export const assessmentSpecPath = assessmentDefinition.specPath; +export const assessmentSummaryPath = assessmentDefinition.summaryPath; +export const assessmentPackagePath = `${assessmentDefinition.packagePath}/package.json`; +export const assessmentMappingPath = assessmentDefinition.mappingPath; +export const checkboxUnitCases = assessmentDefinition.unit.cases; +export const checkboxUnitTestPath = `${assessmentDefinition.packagePath}/${assessmentDefinition.unit.testFiles[0]}`; +export const checkboxNativeAssertionPath = `${assessmentDefinition.packagePath}/${assessmentDefinition.contract.assertionFiles[0]}`; +export const unitMappingFiles = assessmentDefinition.unit.inputs; +export const readUnitMappingFrom = (read: (file: string) => string) => readMapping(read, assessmentDefinition); +export const parseComponentJest = (content: string) => parseJest(content, assessmentDefinition); +export const readControlContractFrom = (read: (file: string) => string) => readContract(read, assessmentDefinition); + +/** Synthetic Jest protocol fixture, not an actual Checkbox test execution. */ +export function checkboxJestFixture(overrides: Record = {}) { + const assertions: { + fullName: string; + ancestorTitles: string[]; + title: string; + status: string; + failureMessages: string[]; + invocations?: number; + retryReasons?: string[]; + }[] = checkboxUnitCases.map(({ fullName }) => { + const win32 = fullName.includes(' accessibility actions '); + const words = fullName.split(' '); + const ancestorTitles = win32 ? ['Checkbox', `${words[1]} accessibility actions`] : ['Checkbox']; + const title = fullName.slice(ancestorTitles.join(' ').length + 1); + return { fullName, ancestorTitles, title, status: 'passed', failureMessages: [] }; + }); + return { + numTotalTestSuites: 1, + numPassedTestSuites: 1, + numFailedTestSuites: 0, + numPendingTestSuites: 0, + numTotalTests: assertions.length, + numPassedTests: assertions.length, + numFailedTests: 0, + numPendingTests: 0, + numTodoTests: 0, + numRuntimeErrorTestSuites: 0, + startTime: Date.parse('2026-10-21T20:00:00.500Z'), + success: true, + wasInterrupted: false, + testResults: [ + { + name: `C:\\synthetic\\${checkboxUnitTestPath.replace(/\//g, '\\')}`, + status: 'passed', + startTime: Date.parse('2026-10-21T20:00:00.600Z'), + endTime: Date.parse('2026-10-21T20:00:01.000Z'), + assertionResults: assertions, + }, + ], + ...overrides, + }; +} + +/** Synthetic existing-package run, including a type suite that must not qualify runtime/type claims. */ +export function componentJestFixture() { + const data = checkboxJestFixture(); + data.testResults.push({ + ...data.testResults[0], + name: data.testResults[0].name.replace('checkbox.test.tsx', 'checkbox.types.test.ts'), + assertionResults: [ + { + fullName: 'Checkbox slot types accepts the public checkbox contract as a slot prop', + ancestorTitles: ['Checkbox slot types'], + title: 'accepts the public checkbox contract as a slot prop', + status: 'passed', + failureMessages: [], + }, + ], + }); + data.numTotalTestSuites++; + data.numPassedTestSuites++; + data.numTotalTests++; + data.numPassedTests++; + return data; +} + +/** Synthetic collector output; this fixture is not native qualification evidence. */ +export function desktopEvidenceFixture(smoke: string, overrides: Record = {}) { + const story = JSON.parse(smoke); + const tests = [ + { + testId: 'navigation:components-checkbox--default', + title: 'Navigate components-checkbox--default', + status: 'passed', + flaky: false, + attempts: [{ attempt: 0, status: 'passed', durationMs: 1, context: { story: 'components-checkbox--default' } }], + }, + ...story.wdio.map( + (entry: { storyId: string; testName?: string; status: string; durationMs: number; skipReason?: string; error?: string }) => { + const name = entry.testName ?? 'default'; + return { + testId: `csf-${reportDigest(JSON.stringify([entry.storyId, name]))}`, + title: `${entry.storyId} / ${name}`, + status: entry.status, + flaky: false, + attempts: [ + { + attempt: 0, + status: entry.status, + durationMs: entry.durationMs, + ...(entry.skipReason ? { reason: entry.skipReason } : {}), + ...(entry.status === 'failed' + ? { + failure: { + failureId: 'test-failure', + kind: 'test', + error: { name: 'Error', message: entry.error ?? 'assertion failed' }, + }, + } + : {}), + context: { endpoint: story.endpoint, story: entry.storyId, test: name }, + }, + ], + }; + }, + ), + ]; + const run = { + schemaVersion: 1, + runId: 'synthetic-desktop-run', + kind: 'tests', + status: story.status, + startedAt: story.startedAt, + finishedAt: story.finishedAt, + durationMs: 1000, + tests, + work: [], + failures: [], + counts: { + selected: tests.length, + executed: tests.filter(({ status }) => status === 'passed' || status === 'failed').length, + passed: tests.filter(({ status }) => status === 'passed').length, + failed: tests.filter(({ status }) => status === 'failed').length, + skipped: tests.filter(({ status }) => status === 'skipped').length, + unsupported: 0, + incomplete: 0, + retries: 0, + flaky: 0, + }, + artifacts: [{ path: 'evidence\\win32\\static\\run.json', description: 'Story smoke results' }], + ...overrides, + }; + const canonical = JSON.stringify(run); + const envelope = JSON.stringify({ schemaVersion: 1, format: 'desktop-test', desktopRun: canonical, storyReport: smoke }); + return { run, canonical, envelope }; +} diff --git a/scripts/src/ci/assessment/controlAssessmentGithub.ts b/scripts/src/ci/assessment/controlAssessmentGithub.ts new file mode 100644 index 0000000000..2e216e201b --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentGithub.ts @@ -0,0 +1,426 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { hex, object, text } from './controlAssessment.ts'; +import { + assessmentArtifactName, + assessmentForBundle, + assessmentWorkflowPath, + assertFresh, + assertTestedInputs, + ensureCommit, + formatCiSummary, + formatCiSpec, + formatCiMapping, + git, + gitFile, + inputDigest, + isReportOnlyUpdate, + parseBundle, + parseReportReference, +} from './controlAssessmentCi.ts'; +import type { CiBundle, CiIdentity, ReportReference } from './controlAssessmentCi.ts'; +import { discoverAssessments, selectAssessment } from './assessmentConfig.ts'; +import type { AssessmentDefinition } from './assessmentConfig.ts'; + +type Run = { + id: number; + run_attempt: number; + event: string; + status: string; + path: string; + head_sha: string; + head_branch: string; + head_repository: { full_name: string } | null; +}; +type PullRequest = { + number: number; + state: string; + base: { sha: string; ref: string; repo: { full_name: string } }; + head: { sha: string; ref: string; repo: { full_name: string } | null }; + merge_commit_sha: string | null; +}; +type Jobs = { jobs: { name: string; conclusion: string | null }[]; total_count: number }; +type Artifacts = { artifacts: { name: string; expired: boolean }[]; total_count: number }; +type GitHubOptions = { token: string; fetchImplementation?: typeof fetch }; + +export class AssessmentGitHub { + private readonly token: string; + private readonly fetchImplementation: typeof fetch; + + constructor({ token, fetchImplementation = fetch }: GitHubOptions) { + this.token = text(token, 'GitHub token'); + this.fetchImplementation = fetchImplementation; + } + + async request(route: string, options: RequestInit = {}): Promise { + if (!route.startsWith('/repos/') && route !== '/graphql') throw new Error('Unexpected assessment API route.'); + const response = await this.fetchImplementation(`https://api.github.com${route}`, { + ...options, + redirect: 'error', + signal: AbortSignal.timeout(30_000), + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + }); + if (!response.ok) throw new Error(`Assessment GitHub API request failed (${response.status}): ${route}.`); + return (await response.json()) as T; + } + + async sourceRun(identity: CiIdentity): Promise { + const run = await this.request(`/repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}`); + if ( + String(run.id) !== identity.runId || + run.run_attempt !== identity.runAttempt || + run.event !== 'pull_request' || + run.status !== 'completed' || + run.path !== assessmentWorkflowPath || + run.head_sha !== identity.headCommit || + run.head_branch !== identity.headRef || + run.head_repository?.full_name !== identity.headRepository + ) { + throw new Error('Assessment evidence does not match the completed source PR workflow attempt.'); + } + return run; + } + + async sourceValidation(identity: CiIdentity): Promise<'success' | 'failure'> { + const matches: Jobs['jobs'] = []; + for (let page = 1; page <= 20; page++) { + const data = await this.request( + `/repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100&page=${page}`, + ); + matches.push(...data.jobs.filter((job) => job.name === 'PR source validation')); + if (page * 100 >= data.total_count) { + if (matches.length !== 1) throw new Error('Source workflow must have exactly one PR source validation job.'); + return matches[0].conclusion === 'success' ? 'success' : 'failure'; + } + } + throw new Error('Source workflow job list exceeds the supported limit.'); + } + + async hasArtifact(repository: string, runId: string, runAttempt: number): Promise { + const name = `${assessmentArtifactName}-${runAttempt}`; + let matches = 0; + for (let page = 1; page <= 20; page++) { + const data = await this.request(`/repos/${repository}/actions/runs/${runId}/artifacts?per_page=100&page=${page}`); + matches += data.artifacts.filter((artifact) => artifact.name === name && !artifact.expired).length; + if (page * 100 >= data.total_count) { + if (matches > 1) throw new Error('Source workflow has ambiguous assessment artifacts.'); + return matches === 1; + } + } + throw new Error('Source workflow artifact list exceeds the supported limit.'); + } +} + +export type Candidate = + | { reportOnly: false; reason: string } + | { + reportOnly: true; + component: string; + packagePath: string; + reference: ReportReference; + summary: string; + spec: string; + mapping: string; + sourceResult: 'success' | 'failure'; + }; + +export type Candidates = + | { reportOnly: false; reason: string } + | { reportOnly: true; candidates: Extract[] }; + +export async function inspectReportUpdate( + root: string, + env: NodeJS.ProcessEnv, + github: AssessmentGitHub, + now = new Date(), + definition = selectAssessment(root), +): Promise { + const assessmentSummaryPath = definition.summaryPath; + const assessmentSpecPath = definition.specPath; + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'PR event'); + const pr = object(event.pull_request, 'PR event pull_request'); + const head = object(pr.head, 'PR head'); + const merge = hex(env.GITHUB_SHA, 40, 'PR merge revision'); + ensureCommit(root, merge); + const parents = git(root, ['rev-list', '--parents', '-n', '1', merge]).split(' '); + if (parents.length !== 3 || parents[1] !== object(pr.base, 'PR base').sha || parents[2] !== head.sha) { + throw new Error('Current CI merge revision does not match the PR event.'); + } + const summary = gitFile(root, merge, assessmentSummaryPath); + if (summary === undefined) return { reportOnly: false, reason: 'No checked-in assessment: run normal source CI.' }; + let reference: ReportReference; + try { + reference = parseReportReference(summary); + assertFresh(reference.generatedAt, now); + } catch (error) { + return { reportOnly: false, reason: `Assessment is not reusable; run normal source CI. ${String(error)}` }; + } + if ( + reference.repository !== env.GITHUB_REPOSITORY || + reference.pullRequest !== pr.number || + reference.headRepository !== object(head.repo, 'PR head repository').full_name || + reference.headRef !== head.ref + ) { + return { reportOnly: false, reason: 'Assessment belongs to a different PR: run normal source CI.' }; + } + if (!isReportOnlyUpdate(root, reference, hex(head.sha, 40, 'PR head revision'), merge)) { + return { + reportOnly: false, + reason: 'Code, contract, test, dependency, configuration, or branch inputs changed: run normal source CI.', + }; + } + await github.sourceRun(reference); + if (!(await github.hasArtifact(reference.repository, reference.runId, reference.runAttempt))) { + throw new Error('Report-only update cannot reuse missing or expired source evidence.'); + } + const spec = gitFile(root, hex(head.sha, 40, 'PR head revision'), assessmentSpecPath); + if (spec === undefined) throw new Error('Report-only update is missing its SPEC assessment snapshot.'); + const mapping = gitFile(root, hex(head.sha, 40, 'PR head revision'), definition.mappingPath); + if (mapping === undefined) throw new Error('Report-only update is missing its reviewed mapping.'); + return { + reportOnly: true, + component: definition.component, + packagePath: definition.packagePath, + reference, + summary, + spec, + mapping, + sourceResult: await github.sourceValidation(reference), + }; +} + +export function verifyReportUpdate(root: string, candidate: Candidate, bundle: CiBundle, now = new Date()): void { + if (!candidate.reportOnly) throw new Error('No report-only candidate was identified.'); + const definition = assessmentForBundle(root, bundle); + if (candidate.component !== definition.component || candidate.packagePath !== definition.packagePath) + throw new Error('Downloaded evidence belongs to a different component.'); + assertFresh(bundle.generatedAt, now); + const reference: ReportReference = { ...bundle.identity, generatedAt: bundle.generatedAt }; + if (JSON.stringify(reference) !== JSON.stringify(candidate.reference)) throw new Error('Downloaded evidence belongs to a different run.'); + if (formatCiSummary(root, bundle) !== candidate.summary) { + throw new Error('Checked-in report is not the trusted rendering of its source evidence.'); + } + if (formatCiSpec(root, bundle) !== candidate.spec) + throw new Error('Checked-in SPEC is not the trusted rendering of its source evidence.'); + if (formatCiMapping(root, bundle) !== candidate.mapping) + throw new Error('Checked-in mapping is not the exact reviewed mapping from the tested source.'); +} + +export async function inspectReportUpdates( + root: string, + env: NodeJS.ProcessEnv, + github: AssessmentGitHub, + now = new Date(), +): Promise { + const definitions = discoverAssessments(root); + if (!definitions.length) return { reportOnly: false, reason: 'No component assessment is configured: run normal source CI.' }; + const candidates: Extract[] = []; + for (const definition of definitions) { + const candidate = await inspectReportUpdate(root, env, github, now, definition); + if (!candidate.reportOnly) return candidate; + const first = candidates[0]; + if ( + first && + (JSON.stringify({ ...first.reference, generatedAt: undefined }) !== + JSON.stringify({ ...candidate.reference, generatedAt: undefined }) || + first.sourceResult !== candidate.sourceResult) + ) + return { reportOnly: false, reason: 'Component reports do not share the same tested workflow attempt: run normal source CI.' }; + candidates.push(candidate); + } + return { reportOnly: true, candidates }; +} + +function assertCompleteComponentSet(root: string, entries: { component?: string; packagePath?: string }[]): AssessmentDefinition[] { + const definitions = discoverAssessments(root); + if ( + !definitions.length || + entries.length !== definitions.length || + definitions.some( + (definition) => + entries.filter((entry) => entry.component === definition.component && entry.packagePath === definition.packagePath).length !== 1, + ) + ) + throw new Error('Assessment evidence must include every configured component exactly once.'); + return definitions; +} + +export function verifyReportUpdates(root: string, candidate: Candidates, bundles: CiBundle[], now = new Date()): void { + if (!candidate.reportOnly) throw new Error('No report-only candidate was identified.'); + assertCompleteComponentSet(root, candidate.candidates); + assertCompleteComponentSet(root, bundles); + for (const entry of candidate.candidates) { + const bundle = bundles.find((bundle) => bundle.component === entry.component && bundle.packagePath === entry.packagePath); + if (!bundle) throw new Error('Missing component evidence.'); + verifyReportUpdate(root, entry, bundle, now); + } +} + +export type Publication = + | { eligible: false; reason: string } + | { + eligible: true; + repository: string; + branch: string; + expectedHead: string; + summary: string; + spec: string; + mapping: string; + summaryPath: string; + specPath: string; + mappingPath: string; + sourceRunId: string; + }; + +export async function preparePublication( + root: string, + bundle: CiBundle, + github: AssessmentGitHub, + expected: { repository: string; runId: string; runAttempt: number; defaultBranch: string }, + now = new Date(), +): Promise { + const definition = assessmentForBundle(root, bundle); + const assessmentSummaryPath = definition.summaryPath; + const assessmentSpecPath = definition.specPath; + const identity = bundle.identity; + if (identity.repository !== expected.repository || identity.runId !== expected.runId || identity.runAttempt !== expected.runAttempt) { + throw new Error('Downloaded bundle is not from the triggering workflow attempt.'); + } + assertFresh(bundle.generatedAt, now); + await github.sourceRun(identity); + assertTestedInputs(root, identity); + const pr = await github.request(`/repos/${identity.repository}/pulls/${identity.pullRequest}`); + if (pr.state !== 'open') return { eligible: false, reason: 'PR is no longer open; no report commit was made.' }; + if (identity.headRepository !== identity.repository || pr.head.repo?.full_name !== identity.repository) { + return { eligible: false, reason: 'Fork PR: the pilot publishes CI artifacts only; no fork branch write is attempted.' }; + } + if (pr.base.repo.full_name !== identity.repository || pr.base.ref !== expected.defaultBranch || pr.head.ref === expected.defaultBranch) { + return { eligible: false, reason: 'The pilot writes only to a PR source branch targeting the default branch, never to main.' }; + } + if (pr.head.sha !== identity.headCommit || pr.head.ref !== identity.headRef) { + return { eligible: false, reason: 'PR head moved since testing; discard the stale publication.' }; + } + if (!pr.merge_commit_sha) return { eligible: false, reason: 'PR merge revision is unavailable or conflicted; resolve and rerun CI.' }; + ensureCommit(root, pr.merge_commit_sha); + const parents = git(root, ['rev-list', '--parents', '-n', '1', pr.merge_commit_sha]).split(' '); + if (parents.length !== 3 || parents[1] !== pr.base.sha || parents[2] !== pr.head.sha) { + return { eligible: false, reason: 'GitHub has not refreshed the current PR merge revision; rerun publication after it is available.' }; + } + if (inputDigest(root, pr.merge_commit_sha) !== identity.inputDigest) { + return { eligible: false, reason: 'Merged source inputs changed since testing; update the branch and rerun CI.' }; + } + const summary = formatCiSummary(root, bundle); + const spec = formatCiSpec(root, bundle); + const mapping = formatCiMapping(root, bundle); + if (Buffer.byteLength(summary) > 100_000) throw new Error('Generated assessment summary exceeds its publication limit.'); + if (Buffer.byteLength(spec) > 100_000) throw new Error('Generated SPEC exceeds its publication limit.'); + if (Buffer.byteLength(mapping) > 200_000) throw new Error('Reviewed mapping exceeds its publication limit.'); + if ( + gitFile(root, identity.headCommit, assessmentSummaryPath) === summary && + gitFile(root, identity.headCommit, assessmentSpecPath) === spec && + gitFile(root, identity.headCommit, definition.mappingPath) === mapping + ) { + return { eligible: false, reason: 'The exact assessment is already present; no duplicate commit was made.' }; + } + return { + eligible: true, + repository: identity.repository, + branch: identity.headRef, + expectedHead: identity.headCommit, + summary, + spec, + mapping, + summaryPath: assessmentSummaryPath, + specPath: assessmentSpecPath, + mappingPath: definition.mappingPath, + sourceRunId: identity.runId, + }; +} + +export async function publishAssessment(publication: Publication, github: AssessmentGitHub): Promise { + if (!publication.eligible) throw new Error('Assessment publication was not validated.'); + return publishAssessments([publication], github); +} + +export async function preparePublications( + root: string, + bundles: CiBundle[], + github: AssessmentGitHub, + expected: Parameters[3], + now = new Date(), +): Promise { + assertCompleteComponentSet(root, bundles); + if (bundles.some((bundle) => JSON.stringify(bundle.identity) !== JSON.stringify(bundles[0].identity))) + throw new Error('Component evidence does not share the same tested workflow attempt.'); + const publications: Publication[] = []; + for (const bundle of bundles) publications.push(await preparePublication(root, bundle, github, expected, now)); + return publications; +} + +export async function publishAssessments(publications: Publication[], github: AssessmentGitHub): Promise { + const first = publications[0]; + if (!first?.eligible || publications.some((publication) => !publication.eligible)) + throw new Error('Every component publication must be validated before an atomic update.'); + const additions: { path: string; contents: string }[] = []; + for (const publication of publications) { + if (!publication.eligible) throw new Error('Assessment publication was not validated.'); + if ( + publication.repository !== first.repository || + publication.branch !== first.branch || + publication.expectedHead !== first.expectedHead || + publication.sourceRunId !== first.sourceRunId + ) + throw new Error('Component publications do not share the same source branch and tested head.'); + additions.push( + { path: publication.summaryPath, contents: Buffer.from(publication.summary).toString('base64') }, + { path: publication.specPath, contents: Buffer.from(publication.spec).toString('base64') }, + { path: publication.mappingPath, contents: Buffer.from(publication.mapping).toString('base64') }, + ); + } + if (new Set(additions.map((addition) => addition.path)).size !== additions.length) + throw new Error('Component publication paths must not overlap.'); + const response = await github.request<{ + data?: { createCommitOnBranch?: { commit?: { oid?: string } } }; + errors?: unknown[]; + }>('/graphql', { + method: 'POST', + body: JSON.stringify({ + query: 'mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }', + variables: { + input: { + branch: { repositoryNameWithOwner: first.repository, branchName: first.branch }, + expectedHeadOid: first.expectedHead, + message: { + headline: 'chore: update component Win32 assessment', + body: `Auto-generated from CI run ${first.sourceRunId}. No source inputs changed.`, + }, + fileChanges: { + additions, + }, + }, + }, + }), + }); + if (response.errors?.length) throw new Error('GitHub rejected the assessment commit; the branch may have moved or prohibit writes.'); + return hex(response.data?.createCommitOnBranch?.commit?.oid, 40, 'Published assessment commit'); +} + +export function readBundle(file: string): CiBundle { + return parseBundle(readFileSync(file, 'utf8')); +} + +export function readBundles(root: string, directory: string): CiBundle[] { + return discoverAssessments(root).map((definition) => { + const componentDirectory = path.join(directory, definition.packagePath, definition.component); + const bundle = readBundle(path.join(componentDirectory, 'bundle.json')); + if (readFileSync(path.join(componentDirectory, 'assessment.mapping.json'), 'utf8') !== formatCiMapping(root, bundle)) + throw new Error('Artifact mapping differs from the exact reviewed mapping in the tested source.'); + return bundle; + }); +} diff --git a/scripts/src/ci/assessment/controlAssessmentSpec.ts b/scripts/src/ci/assessment/controlAssessmentSpec.ts new file mode 100644 index 0000000000..c5a203a665 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentSpec.ts @@ -0,0 +1,174 @@ +import type { ControlAssessment } from './controlAssessment.ts'; + +const begin = '# furn-assessment:begin'; +const end = '# furn-assessment:end'; +const outcomes = ['failed', 'execution-failed', 'insufficient-evidence'] as const; +const provenances = ['missing', 'unverified', 'matched', 'mismatched', 'stale'] as const; +const healths = ['complete', 'stale', 'blocked'] as const; + +type SpecAssessment = { + packageVersion: string; + sourceCommit: string; + testedCommit: string | null; + dirty: boolean; + generatedAt: string; + outcome: (typeof outcomes)[number]; + provenance: (typeof provenances)[number]; + health: (typeof healths)[number]; +}; + +function member(value: unknown, values: readonly T[]): T { + const result = values.find((entry) => entry === value); + if (result === undefined) throw new Error('Unsupported generated SPEC assessment value.'); + return result; +} + +function validate(value: SpecAssessment): void { + if ( + typeof value.packageVersion !== 'string' || + value.packageVersion.length > 100 || + !/^\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?(?:\+[A-Za-z0-9.-]+)?$/.test(value.packageVersion) || + !/^[0-9a-f]{40}$/.test(value.sourceCommit) || + typeof value.dirty !== 'boolean' || + !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(value.generatedAt) || + !Number.isFinite(Date.parse(value.generatedAt)) || + new Date(value.generatedAt).toISOString() !== value.generatedAt + ) { + throw new Error('Invalid generated SPEC assessment identity.'); + } + member(value.outcome, outcomes); + member(value.provenance, provenances); + member(value.health, healths); + if (value.testedCommit !== (value.provenance === 'matched' && !value.dirty ? value.sourceCommit : null)) { + throw new Error('Only matched, clean execution evidence can identify a tested SPEC commit.'); + } +} + +function fields(value: SpecAssessment): [string, string | number | boolean | null][] { + validate(value); + return [ + ['assessment-schema', 1], + ['assessment-package-version', value.packageVersion], + ['assessment-source-commit', value.sourceCommit], + ['assessment-tested-commit', value.testedCommit], + ['assessment-dirty', value.dirty], + ['assessment-target', 'win32-paper-storybook'], + ['assessment-outcome', value.outcome], + ['assessment-health', value.health], + ['assessment-stage', 'not-determined'], + ['assessment-provenance', value.provenance], + ['assessment-generated-at', value.generatedAt], + ['assessment-report', './assessment/ASSESSMENT.md'], + ]; +} + +function block(value: SpecAssessment, eol: string): string { + return [ + begin, + ...fields(value).map(([key, scalar]) => `${key}: ${typeof scalar === 'string' ? `'${scalar}'` : JSON.stringify(scalar)}`), + end, + '', + ].join(eol); +} + +function header(spec: string): { eol: string; close: number } { + const eol = spec.startsWith('---\r\n') ? '\r\n' : '\n'; + if (spec.length > 1_000_000 || !spec.startsWith(`---${eol}`)) throw new Error('SPEC must start with bounded YAML front matter.'); + const delimiter = `${eol}---`; + const match = spec.indexOf(`${delimiter}${eol}`, 3); + const close = match >= 0 ? match + eol.length : spec.endsWith(delimiter) ? spec.length - 3 : -1; + if (close < 0) throw new Error('SPEC YAML front matter is unterminated.'); + return { eol, close }; +} + +export function splitSpecAssessment(spec: string): { contract: string; assessment?: SpecAssessment } { + const { eol, close } = header(spec); + const start = spec.indexOf(begin); + const finish = spec.indexOf(end); + let contract = spec; + let assessment: SpecAssessment | undefined; + if (start !== -1 || finish !== -1) { + const after = finish + end.length + eol.length; + if ( + start < 3 + eol.length || + finish <= start || + after > close || + spec[start - 1] !== '\n' || + spec.slice(finish - eol.length, finish) !== eol || + spec.slice(finish + end.length, after) !== eol || + spec.indexOf(begin, start + begin.length) !== -1 || + spec.indexOf(end, finish + end.length) !== -1 + ) { + throw new Error('Generated SPEC assessment must be one complete front-matter block.'); + } + const raw = spec.slice(start, after); + const values: Record = Object.create(null); + for (const line of raw.split(eol).slice(1, -2)) { + const separator = line.indexOf(': '); + if (separator < 1) throw new Error('Malformed generated SPEC assessment field.'); + const key = line.slice(0, separator); + if (Object.hasOwn(values, key)) throw new Error('Duplicate generated SPEC assessment field.'); + const scalar = line.slice(separator + 2); + values[key] = scalar.startsWith("'") && scalar.endsWith("'") ? scalar.slice(1, -1) : JSON.parse(scalar); + } + if ( + values['assessment-schema'] !== 1 || + values['assessment-target'] !== 'win32-paper-storybook' || + values['assessment-stage'] !== 'not-determined' || + values['assessment-report'] !== './assessment/ASSESSMENT.md' || + typeof values['assessment-package-version'] !== 'string' || + typeof values['assessment-source-commit'] !== 'string' || + (values['assessment-tested-commit'] !== null && typeof values['assessment-tested-commit'] !== 'string') || + typeof values['assessment-dirty'] !== 'boolean' || + typeof values['assessment-generated-at'] !== 'string' + ) { + throw new Error('Invalid generated SPEC assessment schema or identity.'); + } + assessment = { + packageVersion: values['assessment-package-version'], + sourceCommit: values['assessment-source-commit'], + testedCommit: values['assessment-tested-commit'], + dirty: values['assessment-dirty'], + generatedAt: values['assessment-generated-at'], + outcome: member(values['assessment-outcome'], outcomes), + provenance: member(values['assessment-provenance'], provenances), + health: member(values['assessment-health'], healths), + }; + if (block(assessment, eol) !== raw) throw new Error('Generated SPEC assessment is not its exact canonical rendering.'); + contract = spec.slice(0, start) + spec.slice(after); + } + if (/^assessment-/m.test(contract.slice(0, header(contract).close))) { + throw new Error('Assessment fields must appear only in the generated SPEC assessment block.'); + } + return { contract, assessment }; +} + +export function updateSpecAssessment(spec: string, assessment: SpecAssessment): string { + const { contract } = splitSpecAssessment(spec); + const { eol, close } = header(contract); + return contract.slice(0, close) + block(assessment, eol) + contract.slice(close); +} + +export function specAssessmentFor(report: ControlAssessment, packageVersion: string): SpecAssessment { + if ( + report.target.endpoint !== 'win32' || + report.target.renderer !== 'paper' || + report.target.host !== 'storybook' || + report.evidenceSupportedStatus !== null || + report.approvedStatus !== null + ) { + throw new Error('The SPEC assessment adapter supports only unassigned Win32/Paper maturity.'); + } + const value: SpecAssessment = { + packageVersion, + sourceCommit: report.target.sourceCommit, + testedCommit: report.evidence.status === 'matched' && !report.target.workingTreeDirty ? report.target.sourceCommit : null, + dirty: report.target.workingTreeDirty, + generatedAt: report.generatedAt, + outcome: member(report.assessmentOutcome, outcomes), + provenance: member(report.evidence.status, provenances), + health: member(report.assessmentHealth, healths), + }; + validate(value); + return value; +} diff --git a/scripts/src/ci/assessment/controlAssessmentUnit.test.ts b/scripts/src/ci/assessment/controlAssessmentUnit.test.ts new file mode 100644 index 0000000000..4f0d02d44d --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentUnit.test.ts @@ -0,0 +1,852 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { assessControl, formatAssessmentMarkdown, parseNativeReport, readControlContract, reportDigest } from './controlAssessment.ts'; +import type { AssessmentOptions } from './controlAssessment.ts'; +import { + assessmentDefinition, + assessmentMappingPath, + checkboxNativeAssertionPath, + checkboxJestFixture, + componentJestFixture, + checkboxUnitCases, + checkboxUnitTestPath, + parseComponentJest, + readUnitMappingFrom, + unitMappingFiles, +} from './controlAssessmentFixtures.ts'; +import { specAssessmentFor } from './controlAssessmentSpec.ts'; +import { parseUnitContext } from './controlAssessmentUnit.ts'; +import { runUnitTests, runJestWithUnitCapture } from './controlAssessmentUnitExecution.ts'; +import { parseAssessmentDefinition, readPackageAssessments, unitCaptureDirectory } from './assessmentConfig.ts'; +import { readUnitMappingFrom as readConfiguredMapping } from './controlAssessmentUnit.ts'; + +const root = fileURLToPath(new URL('../../../../', import.meta.url)); +const read = (file: string) => readFileSync(path.join(root, file), 'utf8'); +const contract = readControlContract(root); +const mapping = readUnitMappingFrom(read); +const sourceCommit = 'a'.repeat(40); +const now = new Date('2026-10-21T20:00:03.000Z'); +const directories: string[] = []; + +function options(data = checkboxJestFixture()): AssessmentOptions { + const report = JSON.stringify(data); + return { + contract, + sourceCommit, + workingTreeDirty: false, + now, + maxAgeHours: 24, + unit: { + mapping, + report, + expectedBuildId: 'synthetic-unit-build', + context: parseUnitContext({ + schemaVersion: 1, + platform: 'macos', + sourceCommit, + workingTreeDirty: false, + buildId: 'synthetic-unit-build', + contractDigest: contract.digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt: '2026-10-21T20:00:00.000Z', + finishedAt: '2026-10-21T20:00:02.000Z', + exitCode: 0, + }), + }, + }; +} + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +describe('Checkbox JavaScript unit evidence', () => { + it('rejects unsafe paths, unlisted contract sources and unfingerprinted or type-only runtime selections', () => { + for (const mutate of [ + (value: typeof assessmentDefinition) => { + value.componentPath = '../outside'; + }, + (value: typeof assessmentDefinition) => { + value.contract.requirements['CBX-001'].checks[0].source = 'unlisted.md'; + }, + (value: typeof assessmentDefinition) => { + value.unit.testFiles = ['src/unreviewed.test.tsx']; + }, + (value: typeof assessmentDefinition) => { + value.unit.testFiles = ['src/checkbox.types.test.ts']; + }, + ]) { + const value = JSON.parse(read(assessmentMappingPath)); + mutate(value); + assert.throws(() => + parseAssessmentDefinition( + value, + assessmentDefinition.packagePath, + assessmentDefinition.packageName, + `${assessmentDefinition.componentPath}/assessment.mapping.json`, + ), + ); + } + }); + + it('requires the checklist to retain every SPEC requirement, including uncovered ones', () => { + for (const change of ['missing', 'extra'] as const) { + const definition = structuredClone(assessmentDefinition); + if (change === 'missing') delete definition.contract.requirements['CBX-006']; + else definition.contract.requirements['CBX-007'] = definition.contract.requirements['CBX-006']; + assert.throws(() => readControlContract(root, definition), /every SPEC requirement/); + } + }); + + it('retains the complete requirement/check set when no tests have been mapped', () => { + const raw = JSON.parse(read(assessmentMappingPath), (key, value) => (key === 'tests' ? [] : value)); + const definition = parseAssessmentDefinition( + raw, + assessmentDefinition.packagePath, + assessmentDefinition.packageName, + `${assessmentDefinition.componentPath}/assessment.mapping.json`, + ); + const report = assessControl({ ...options(), contract: { ...contract, definition } }); + assert.deepEqual( + report.requirements.map(({ id, checks }) => ({ id, checks: checks.map(({ id }) => id) })), + Object.entries(assessmentDefinition.contract.requirements).map(([id, { checks }]) => ({ id, checks: checks.map(({ id }) => id) })), + ); + assert.ok( + report.requirements.every( + ({ checks, outcome }) => checks.every(({ outcome }) => outcome === 'uncovered') && outcome === 'insufficient-evidence', + ), + ); + }); + + it('shows precise shared checks and uncovered checks without turning partial coverage into requirement completion', () => { + const report = assessControl(options()); + const find = (requirement: string, id: string) => + report.requirements.find(({ id }) => id === requirement)!.checks.find((c) => c.id === id)!; + assert.equal(find('CBX-001', 'default-unchecked').outcome, 'passed'); + assert.deepEqual(find('CBX-001', 'default-unchecked'), find('CBX-004', 'default-unchecked')); + assert.equal(find('CBX-001', 'default-standard-variant').outcome, 'uncovered'); + assert.equal(find('CBX-001', 'reject-children').outcome, 'uncovered'); + assert.equal(find('CBX-004', 'native-checkbox-role').outcome, 'not-run'); + assert.equal(find('CBX-004', 'native-toggle-dispatch').outcome, 'uncovered'); + assert.ok(report.requirements.every(({ outcome }) => outcome !== 'passed')); + assert.match(formatAssessmentMarkdown(report), /SPEC requirement \| Specific check/); + assert.match(formatAssessmentMarkdown(report), /No reviewed test mapping/); + }); + + it('requires every linked test to qualify and does not qualify a multi-test check from one passing case', () => { + const data = checkboxJestFixture(); + const test = data.testResults[0].assertionResults.find(({ fullName }) => fullName.includes('falls back to the label text'))!; + test.status = 'pending'; + data.numPassedTests--; + data.numPendingTests++; + const report = assessControl(options(data)); + const check = report.requirements.find(({ id }) => id === 'CBX-003')!.checks.find(({ id }) => id === 'hidden-label')!; + assert.equal(check.tests.length, 2); + assert.deepEqual( + check.tests.map(({ evidenceOutcome }) => evidenceOutcome), + ['passed', 'skipped'], + ); + assert.equal(check.outcome, 'skipped'); + }); + + it('rejects conflicting shared IDs, duplicate links, and JS evidence attached to a native claim', () => { + for (const mutate of [ + (value: typeof assessmentDefinition) => { + value.contract.requirements['CBX-004'].checks.find(({ id }) => id === 'default-unchecked')!.description = 'Conflicting definition'; + }, + (value: typeof assessmentDefinition) => { + const check = value.contract.requirements['CBX-001'].checks[0]; + check.tests.push(check.tests[0]); + }, + (value: typeof assessmentDefinition) => { + value.contract.requirements['CBX-001'].checks[0].evidence = 'native'; + }, + ]) { + const value = JSON.parse(read(assessmentMappingPath)); + mutate(value); + assert.throws(() => + parseAssessmentDefinition( + value, + assessmentDefinition.packagePath, + assessmentDefinition.packageName, + `${value.componentPath}/assessment.mapping.json`, + ), + ); + } + }); + + it('maps Checkbox from a complete existing package report without counting its type suite', () => { + const parsed = parseComponentJest(JSON.stringify(componentJestFixture())); + assert.equal(parsed.packageSuites, 2); + assert.equal(parsed.packageCounts.passed, 43); + assert.equal(parsed.counts.passed, 42); + assert.equal(parsed.entries.length, 42); + const report = assessControl(options(componentJestFixture())); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.requirements[5].outcome, 'insufficient-evidence'); + assert.equal(specAssessmentFor(report, '0.7.0').testedCommit, null); + }); + + it('counts duplicate titles in unrelated parameterized suites without using them as Checkbox identities', () => { + for (const failed of [false, true]) { + const data = componentJestFixture(); + const other = data.testResults[1]; + other.name = 'packages/agentic/components/src/components/menu-button/menu-button.test.tsx'; + other.assertionResults.push({ ...other.assertionResults[0], failureMessages: [] }); + data.numTotalTests++; + data.numPassedTests++; + if (failed) { + other.assertionResults[1].status = 'failed'; + other.assertionResults[1].failureMessages.push('Synthetic unrelated duplicate-title failure'); + other.status = 'failed'; + data.success = false; + data.numPassedTests--; + data.numFailedTests++; + data.numPassedTestSuites--; + data.numFailedTestSuites++; + } + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.checks.length, 42); + assert.equal(report.unitEvidence.unmappedCases.length, 0); + assert.deepEqual(report.unitEvidence.packageCounts, { passed: failed ? 43 : 44, failed: failed ? 1 : 0, skipped: 0 }); + assert.equal(report.unitEvidence.outcome, failed ? 'execution-failed' : 'passed'); + } + }); + + it('retains a failed enclosing package run without attributing another suite failure to Checkbox', () => { + const data = componentJestFixture(); + data.success = false; + data.numPassedTests--; + data.numFailedTests++; + data.numPassedTestSuites--; + data.numFailedTestSuites++; + data.testResults[1].status = 'failed'; + data.testResults[1].assertionResults[0].status = 'failed'; + data.testResults[1].assertionResults[0].failureMessages.push('Synthetic other-suite failure'); + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.packageCounts!.failed, 1); + assert.equal(report.unitEvidence.caseCounts.failed, 0); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('reviews all 42 exact cases but excludes other-platform actions and macOS focus from Win32 coverage', () => { + assert.equal(mapping.current, true); + assert.equal(checkboxUnitCases.length, 42); + assert.equal(new Set(checkboxUnitCases.map(({ fullName }) => fullName)).size, 42); + const report = assessControl(options()); + assert.equal(report.schemaVersion, 2); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.equal(report.unitEvidence.checks.filter(({ requirements }) => requirements.length === 0).length, 17); + assert.deepEqual( + report.requirements.map(({ outcome }) => outcome), + ['partial', 'partial', 'partial', 'partial', 'partial', 'insufficient-evidence'], + ); + assert.deepEqual(report.requirements[5].unitChecks, []); + assert.equal(report.evidence.status, 'missing'); + assert.equal(report.assessmentOutcome, 'insufficient-evidence'); + assert.equal(report.evidenceSupportedStatus, null); + assert.equal(report.approvedStatus, null); + assert.equal(specAssessmentFor(report, '0.7.0').testedCommit, null); + const markdown = formatAssessmentMarkdown(report); + assert.match(markdown, /macOS Jest mock/); + assert.match(markdown, /not native Win32/); + assert.match(markdown, /Not mapped to Win32/); + assert.match(markdown, /0 \/ 0 mapped/); + }); + + it('pins assertion and mock configuration bytes, not merely familiar test names', () => { + for (const changed of unitMappingFiles) { + assert.equal( + readUnitMappingFrom((file) => read(file) + (file === changed ? '\n// changed assertion or configuration\n' : '')).current, + false, + ); + } + assert.equal(readUnitMappingFrom((file) => read(file).replace(/\r?\n/g, '\r\n')).current, true); + assert.equal( + readUnitMappingFrom((file) => + file.endsWith('/package.json') ? JSON.stringify({ ...JSON.parse(read(file)), furn: { jestPlatform: 'win32' } }) : read(file), + ).current, + false, + ); + const input = options(); + input.unit!.mapping = { ...mapping, current: false }; + const report = assessControl(input); + assert.equal(report.assessmentHealth, 'stale'); + assert.equal(report.unitEvidence.outcome, 'stale'); + assert.ok(report.unitEvidence.checks.every(({ evidenceOutcome }) => evidenceOutcome === 'stale')); + }); + + it('invalidates reviewed evidence when package-owned assertion scopes change', () => { + const changed = (file: string) => { + if (file !== assessmentMappingPath) return read(file); + const value = JSON.parse(read(file)); + value.contract.requirements['CBX-001'].checks[2].description = 'Changed claim that has not been reviewed.'; + return JSON.stringify(value); + }; + assert.equal(readUnitMappingFrom(changed).current, false); + }); + + it('keeps a passing suite without execution context unverified', () => { + const input = options(); + input.unit!.context = undefined; + const report = assessControl(input); + assert.equal(report.unitEvidence.status, 'unverified'); + assert.equal(report.unitEvidence.caseCounts.passed, 42); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + for (const missing of ['dirty target', 'dirty execution', 'build', 'age'] as const) { + it(`does not qualify unit evidence with ${missing}`, () => { + const input = options(); + if (missing === 'dirty target') input.workingTreeDirty = true; + if (missing === 'dirty execution') input.unit!.context!.workingTreeDirty = true; + if (missing === 'build') input.unit!.expectedBuildId = undefined; + if (missing === 'age') input.maxAgeHours = undefined; + assert.equal(assessControl(input).unitEvidence.status, 'unverified'); + }); + } + + for (const field of ['sourceCommit', 'buildId', 'contractDigest', 'mappingDigest', 'reportSha256', 'startedAt', 'finishedAt'] as const) { + it(`blocks mismatched unit ${field}`, () => { + const input = options(); + const context = input.unit!.context!; + if (field === 'startedAt') context.startedAt = '2026-10-21T20:00:00.900Z'; + else if (field === 'finishedAt') context.finishedAt = '2026-10-21T20:01:00.000Z'; + else context[field] = field === 'buildId' ? 'different-build' : 'f'.repeat(field === 'sourceCommit' ? 40 : 64); + const report = assessControl(input); + assert.equal(report.unitEvidence.status, 'mismatched'); + assert.equal(report.assessmentHealth, 'blocked'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + } + + it('preserves stale execution and contract mapping state independently', () => { + const input = options(); + input.now = new Date('2026-10-23T20:00:00.000Z'); + assert.equal(assessControl(input).unitEvidence.status, 'stale'); + input.now = now; + input.contract = { ...contract, mappingCurrent: false }; + const report = assessControl(input); + assert.equal(report.assessmentHealth, 'stale'); + assert.ok(report.unitEvidence.checks.every(({ evidenceOutcome }) => evidenceOutcome === 'stale')); + }); + + it('does not invent a passing default assertion when it is missing or replaced by an unmapped name', () => { + for (const replace of [false, true]) { + const data = checkboxJestFixture(); + if (replace) { + Object.assign(data.testResults[0].assertionResults[0], { fullName: 'Checkbox different assertion', title: 'different assertion' }); + } else { + data.testResults[0].assertionResults.shift(); + data.numTotalTests--; + data.numPassedTests--; + } + const report = assessControl(options(data)); + assert.equal(report.requirements[0].outcome, 'insufficient-evidence'); + assert.equal(report.unitEvidence.checks[0].recordedOutcome, 'not-run'); + assert.equal(report.unitEvidence.unmappedCases.length, replace ? 1 : 0); + } + }); + + it('keeps skipped and todo assertions distinct from passing execution', () => { + for (const status of ['pending', 'todo']) { + const data = checkboxJestFixture(); + data.testResults[0].assertionResults[0].status = status; + data.testResults[0].status = status === 'pending' ? 'focused' : 'passed'; + data.numPassedTests--; + if (status === 'todo') data.numTodoTests++; + else data.numPendingTests++; + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.caseCounts.skipped, 1); + assert.equal(report.unitEvidence.checks[0].evidenceOutcome, 'skipped'); + assert.equal(report.requirements[0].outcome, 'insufficient-evidence'); + } + }); + + it('preserves partially skipped unrelated suites using the Jest focused status', () => { + const data = componentJestFixture(); + const other = data.testResults[1]; + other.status = 'focused'; + other.assertionResults.push({ + fullName: 'Other pending assertion', + ancestorTitles: ['Other'], + title: 'pending assertion', + status: 'pending', + failureMessages: [], + }); + data.numTotalTests++; + data.numPendingTests++; + const report = assessControl(options(data)); + assert.deepEqual(report.unitEvidence.packageCounts, { passed: 43, failed: 0, skipped: 1 }); + assert.equal(report.unitEvidence.caseCounts.skipped, 0); + assert.equal(report.unitEvidence.outcome, 'passed'); + }); + + it('preserves retry-success history without qualifying its enclosing execution', () => { + for (const metadata of [ + { invocations: 2, retryReasons: ['Prior synthetic '] }, + { invocations: 2 }, + { retryReasons: ['Prior synthetic '] }, + ]) { + const data = checkboxJestFixture(); + Object.assign(data.testResults[0].assertionResults[0], metadata); + const report = assessControl(options(data)); + const check = report.unitEvidence.checks[0]; + assert.equal(check.recordedOutcome, 'passed'); + assert.equal(check.evidenceOutcome, 'unverified'); + assert.equal(check.invocations, metadata.invocations ?? null); + assert.deepEqual(check.retryReasons, metadata.retryReasons ?? []); + assert.equal(check.retries, 1); + assert.equal(check.flaky, true); + assert.deepEqual(report.unitEvidence.retryCounts, { retries: 1, flaky: 1 }); + assert.equal(report.unitEvidence.runFailed, false); + assert.equal(report.unitEvidence.runRetried, true); + assert.equal(report.unitEvidence.outcome, 'retried'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + const markdown = formatAssessmentMarkdown(report); + assert.match(markdown, /Retried or flaky unit execution/); + if (metadata.retryReasons) assert.ok(markdown.includes('Prior synthetic \\')); + } + }); + + it('retains unrelated package retries without attributing them to Checkbox cases', () => { + const data = componentJestFixture(); + data.testResults[1].assertionResults[0].invocations = 2; + const report = assessControl(options(data)); + assert.deepEqual(report.unitEvidence.retryCounts, { retries: 0, flaky: 0 }); + assert.deepEqual(report.unitEvidence.packageRetryCounts, { retries: 1, flaky: 1 }); + assert.equal(report.unitEvidence.outcome, 'retried'); + assert.ok(report.unitEvidence.checks.every(({ retries, evidenceOutcome }) => retries === 0 && evidenceOutcome === 'unverified')); + }); + + it('does not call an all-skipped suite passed', () => { + const data = checkboxJestFixture(); + data.testResults[0].status = 'skipped'; + data.numPassedTestSuites = 0; + data.numPendingTestSuites = 1; + data.testResults[0].startTime = 0; + data.testResults[0].endTime = 0; + for (const entry of data.testResults[0].assertionResults) entry.status = 'pending'; + data.numPassedTests = 0; + data.numPendingTests = 42; + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.outcome, 'skipped'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('blocks mapped failures and never accepts other passes from the enclosing failed unit command', () => { + const data = checkboxJestFixture(); + const entry = data.testResults[0].assertionResults.find(({ fullName }) => fullName.includes('disables interaction'))!; + entry.status = 'failed'; + entry.failureMessages = ['Synthetic assertion failure']; + entry.invocations = 2; + entry.retryReasons = ['Prior synthetic assertion failure']; + data.success = false; + data.testResults[0].status = 'failed'; + data.numPassedTestSuites = 0; + data.numFailedTestSuites = 1; + data.numPassedTests--; + data.numFailedTests++; + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.outcome, 'failed'); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[4].outcome, 'insufficient-evidence'); + assert.match(formatAssessmentMarkdown(report), /Synthetic assertion failure/); + assert.equal(specAssessmentFor(report, '0.7.0').outcome, 'failed'); + }); + + it('retains runner/parent-command failure even if all recorded assertions passed', () => { + for (const failure of ['interrupted', 'unsuccessful', 'parent command', 'exit code']) { + const data = checkboxJestFixture(); + if (failure === 'interrupted') data.wasInterrupted = true; + if (failure === 'unsuccessful') data.success = false; + const input = options(data); + if (failure === 'parent command') input.unit!.executionFailure = 'Declared workspace script failed.'; + if (failure === 'exit code') input.unit!.context!.exitCode = 1; + const report = assessControl(input); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.assessmentOutcome, 'execution-failed'); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + } + }); + + it('does not let passing unit evidence hide a bound native failure', () => { + const input = options(); + const native = JSON.stringify({ + endpoint: 'win32', + platformManifestDigest: 'b'.repeat(64), + status: 'failed', + tests: [ + { + storyId: 'components-checkbox--default', + testName: 'toggles native checked state in both directions', + status: 'failed', + durationMs: 1, + }, + ], + }); + input.report = parseNativeReport(JSON.parse(native)); + input.reportSha256 = reportDigest(native); + input.expectedBuildId = 'synthetic-native-build'; + input.context = { + schemaVersion: 1, + reportSha256: input.reportSha256, + sourceCommit, + buildId: input.expectedBuildId, + contractDigest: contract.digest, + platformManifestDigest: 'b'.repeat(64), + endpoint: 'win32', + renderer: 'paper', + targetId: 'agenticstorybook-win32', + finishedAt: '2026-10-21T20:00:02.000Z', + physicalClick: true, + }; + const report = assessControl(input); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.assessmentOutcome, 'failed'); + assert.equal(report.requirements[1].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[3].outcome, 'blocked-by-failure'); + assert.equal(report.requirements[0].outcome, 'partial'); + }); + + it('records startup/runtime failure without manufacturing completed assertion timestamps', () => { + const data = checkboxJestFixture(); + Object.assign(data, { + success: false, + numRuntimeErrorTestSuites: 1, + numPassedTestSuites: 0, + numFailedTestSuites: 1, + numTotalTests: 0, + numPassedTests: 0, + }); + Object.assign(data.testResults[0], { status: 'failed', startTime: 0, endTime: 0, assertionResults: [] }); + const report = assessControl(options(data)); + assert.equal(report.unitEvidence.outcome, 'execution-failed'); + assert.equal(report.unitEvidence.caseCounts.passed, 0); + assert.ok(report.requirements.every(({ outcome }) => outcome === 'insufficient-evidence')); + }); + + it('rejects invalid schemas, type-suite substitution, duplicates, contradictory counters, and oversized reports', () => { + const mutations = [ + (data: ReturnType) => { + data.testResults[0].name = 'checkbox.types.test.ts'; + }, + (data: ReturnType) => { + data.testResults.push(data.testResults[0]); + }, + (data: ReturnType) => { + data.testResults[0].assertionResults.push(data.testResults[0].assertionResults[0]); + }, + (data: ReturnType) => { + data.numPassedTests = 900; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].status = 'unknown'; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].title = 'wrong identity'; + }, + (data: ReturnType) => { + data.testResults[0].assertionResults[0].failureMessages = ['contradiction']; + }, + (data: ReturnType) => { + data.numRuntimeErrorTestSuites = 1; + }, + (data: ReturnType) => { + data.testResults[0].endTime = 1; + }, + (data: ReturnType) => { + data.numFailedTestSuites = 1; + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { invocations: -1 }); + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { retryReasons: 'invalid' }); + }, + (data: ReturnType) => { + Object.assign(data.testResults[0].assertionResults[0], { retryReasons: [null] }); + }, + (data: ReturnType) => { + data.testResults[0].status = 'focused'; + }, + (data: ReturnType) => { + data.testResults[0].status = 'skipped'; + data.numPassedTestSuites = 0; + data.numPendingTestSuites = 1; + }, + ]; + for (const mutate of mutations) { + const data = checkboxJestFixture(); + mutate(data); + assert.throws(() => parseComponentJest(JSON.stringify(data))); + } + assert.throws(() => parseComponentJest('{}')); + assert.throws(() => parseComponentJest(' '.repeat(1_000_001)), /size limit/); + const input = options(); + assert.throws(() => parseUnitContext({ ...input.unit!.context, platform: 'invalid' }), /platform/); + assert.throws(() => parseUnitContext({ ...input.unit!.context, exitCode: -1 }), /exit code/); + assert.throws(() => parseUnitContext({ ...input.unit!.context, finishedAt: '2026-10-20T00:00:00.000Z' }), /before/); + input.unit!.report = undefined; + assert.throws(() => assessControl(input), /requires a unit report/); + }); +}); + +describe('unit execution capture', () => { + it('leaves uncaptured and other-platform package execution unchanged and calls the existing runner once', async () => { + const args = ['--runTestsByPath', 'existing.test.ts']; + for (const [platform, env] of [ + ['macos', {}], + ['windows', { FURN_ASSESSMENT_UNIT_CAPTURE: 'unused' }], + ] as const) { + let invocations = 0; + const code = await runJestWithUnitCapture( + path.join(root, assessmentDefinition.packagePath), + platform, + args, + async (forwarded) => { + invocations++; + assert.deepEqual(forwarded, args); + return 7; + }, + env, + ); + assert.equal(code, 7); + assert.equal(invocations, 1); + } + }); + + function fixture(mode = 'passed') { + const directory = mkdtempSync(path.join(tmpdir(), 'furn-unit-capture-')); + directories.push(directory); + const put = (file: string, content: string) => { + mkdirSync(path.dirname(path.join(directory, file)), { recursive: true }); + writeFileSync(path.join(directory, file), content); + }; + for (const file of [...unitMappingFiles, assessmentMappingPath, 'packages/agentic/components/package.json']) put(file, read(file)); + put(checkboxNativeAssertionPath, read(checkboxNativeAssertionPath)); + for (const file of [ + 'SPEC.md', + 'spec/source.json', + 'spec/interaction.md', + 'spec/accessibility.md', + 'spec/tokens.yaml', + 'spec/usage.md', + 'checkbox.stories.tsx', + ]) { + const relative = path.posix.join(path.posix.dirname(checkboxUnitTestPath), file); + put(relative, read(relative)); + } + put('package.json', '{"packageManager":"yarn@0.0.0"}'); + put('.gitignore', 'apps/storybook/artifacts/\n'); + put('source.ts', 'original\n'); + put('fixture-unit.json', JSON.stringify({ ...checkboxJestFixture(), fixtureMode: mode })); + // This synthetic package-manager fixture exercises capture, not component execution. + put( + '.yarn/releases/yarn-0.0.0.cjs', + ` +const fs = require('node:fs'); +const path = require('node:path'); +const data = JSON.parse(fs.readFileSync('fixture-unit.json', 'utf8')); +if (data.fixtureMode === 'missing') process.exit(0); +const now = Date.now(); +data.startTime = now; +data.testResults[0].startTime = now; +data.testResults[0].endTime = now; +const destination = process.argv[process.argv.indexOf('--outputFile') + 1]; +fs.writeFileSync(destination, JSON.stringify(data)); +if (data.fixtureMode === 'changed') fs.writeFileSync('source.ts', 'changed\\n'); +if (data.fixtureMode === 'failed') process.exitCode = 1; +`, + ); + const git = (...args: string[]) => execFileSync('git', args, { cwd: directory, encoding: 'utf8' }).trim(); + git('init', '-q'); + git('config', 'user.name', 'Fixture'); + git('config', 'user.email', 'fixture@example.invalid'); + git('config', 'core.autocrlf', 'false'); + git('config', 'commit.gpgsign', 'false'); + git('add', '.'); + git('commit', '-qm', 'Synthetic unit capture inputs'); + return { directory, output: path.join(directory, 'apps', 'storybook', 'artifacts', 'unit') }; + } + + it('discovers a second control from package data and binds both to one unchanged Jest invocation', async () => { + const f = fixture(); + const packageRoot = path.join(f.directory, assessmentDefinition.packagePath); + const put = (file: string, value: string) => { + mkdirSync(path.dirname(path.join(f.directory, file)), { recursive: true }); + writeFileSync(path.join(f.directory, file), value); + }; + const original = JSON.parse(read(assessmentMappingPath)); + const second = structuredClone(original); + second.component = 'synthetic-toggle'; + second.displayName = 'Synthetic Toggle'; + second.componentPath = 'src/components/synthetic-toggle'; + second.contract.nativeCheck = { + storyId: 'components-synthetic-toggle--default', + testName: 'changes native state', + requiresPhysicalClick: true, + }; + second.contract.requirements = Object.fromEntries( + Object.entries(second.contract.requirements).map(([id, value]) => [id.replace('CBX-', 'TGL-'), value]), + ); + second.unit.testFiles = ['src/components/synthetic-toggle/toggle.test.tsx']; + second.unit.inputs[0] = `${assessmentDefinition.packagePath}/${second.unit.testFiles[0]}`; + for (const requirement of Object.values(second.contract.requirements) as { checks: { tests: unknown[] }[] }[]) + for (const check of requirement.checks) check.tests = []; + second.contract.requirements['TGL-002'].checks = [ + { + id: 'synthetic-state', + description: 'Synthetic JS state assertion.', + source: 'SPEC.md#requirements', + evidence: 'unit', + tests: [{ kind: 'unit', fullName: 'Synthetic Toggle changes state' }], + }, + ]; + second.unit.excludedCases = []; + const secondMapping = `${assessmentDefinition.packagePath}/${second.componentPath}/assessment.mapping.json`; + for (const file of second.contract.files) { + let content = read(`${assessmentDefinition.packagePath}/${assessmentDefinition.componentPath}/${file}`).replaceAll('CBX-', 'TGL-'); + if (file === 'spec/source.json') content = JSON.stringify({ ...JSON.parse(content), component: second.component }); + put(`${assessmentDefinition.packagePath}/${second.componentPath}/${file}`, content); + } + put(second.unit.inputs[0], '// Synthetic test-selection fixture, not a component execution.\n'); + put(secondMapping, JSON.stringify(second)); + const manifestFile = `${assessmentDefinition.packagePath}/package.json`; + const manifest = JSON.parse(read(manifestFile)); + manifest.furn.assessments.push(`${second.componentPath}/assessment.mapping.json`); + put(manifestFile, JSON.stringify(manifest)); + let definitions = readPackageAssessments(f.directory, packageRoot); + const secondDefinition = definitions.find(({ component }) => component === second.component)!; + second.contract.reviewedDigest = readControlContract(f.directory, secondDefinition).digest; + second.unit.reviewedDigest = readConfiguredMapping( + (file) => readFileSync(path.join(f.directory, file), 'utf8'), + secondDefinition, + ).digest; + put(secondMapping, JSON.stringify(second)); + definitions = readPackageAssessments(f.directory, packageRoot); + execFileSync('git', ['add', '.'], { cwd: f.directory }); + execFileSync('git', ['commit', '-qm', 'Synthetic second control configuration'], { cwd: f.directory }); + const testedCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: f.directory, encoding: 'utf8' }).trim(); + mkdirSync(f.output, { recursive: true }); + writeFileSync( + path.join(f.output, 'unit-collection-start.json'), + JSON.stringify({ + schemaVersion: 1, + identity: { testedCommit }, + buildId: 'synthetic-shared-run', + startedAt: new Date().toISOString(), + }), + ); + const args = ['--runTestsByPath', ...definitions.flatMap(({ unit }) => unit.testFiles)]; + let calls = 0; + const code = await runJestWithUnitCapture( + packageRoot, + 'macos', + args, + async (forwarded) => { + calls++; + assert.deepEqual(forwarded.slice(0, args.length), args); + const data = checkboxJestFixture(); + const time = Date.now(); + data.startTime = time; + Object.assign(data.testResults[0], { startTime: time, endTime: time }); + data.testResults.push({ + ...data.testResults[0], + name: `${assessmentDefinition.packagePath}/${second.unit.testFiles[0]}`, + assertionResults: [ + { + fullName: 'Synthetic Toggle changes state', + ancestorTitles: ['Synthetic Toggle'], + title: 'changes state', + status: 'passed', + failureMessages: [], + }, + ], + }); + data.numTotalTestSuites++; + data.numPassedTestSuites++; + data.numTotalTests++; + data.numPassedTests++; + writeFileSync(forwarded[forwarded.indexOf('--outputFile') + 1], JSON.stringify(data)); + return 0; + }, + { FURN_ASSESSMENT_UNIT_CAPTURE: f.output }, + ); + assert.equal(code, 0); + assert.equal(calls, 1); + for (const definition of definitions) { + const capture = JSON.parse(readFileSync(path.join(unitCaptureDirectory(f.output, definition), 'unit-execution.json'), 'utf8')); + const report = assessControl({ + contract: readControlContract(f.directory, definition), + sourceCommit: testedCommit, + workingTreeDirty: false, + now: new Date(), + maxAgeHours: 24, + unit: { + ...capture, + expectedBuildId: 'synthetic-shared-run', + mapping: readConfiguredMapping((file) => readFileSync(path.join(f.directory, file), 'utf8'), definition), + }, + }); + assert.equal(report.component, definition.component); + assert.equal(report.unitEvidence.status, 'matched'); + assert.equal(report.unitEvidence.outcome, 'passed'); + assert.equal(report.unitEvidence.checks.length, definition.unit.cases.length); + assert.equal(report.evidenceSupportedStatus, null); + } + assert.equal(calls, 1); + }); + + it('captures exact bytes and source before execution, with a fresh output directory for each run', () => { + const f = fixture(); + const first = runUnitTests(f.directory, f.output); + const second = runUnitTests(f.directory, f.output); + assert.equal(first.exitCode, 0); + assert.equal(first.context!.workingTreeDirty, false); + assert.equal(first.context!.platform, 'macos'); + assert.equal(first.context!.reportSha256, reportDigest(first.report!)); + assert.notEqual(first.context!.buildId, second.context!.buildId); + assert.equal(readdirSync(f.output).length, 2); + const report = assessControl({ + contract: readControlContract(f.directory), + sourceCommit: first.context!.sourceCommit, + workingTreeDirty: false, + now: new Date(), + maxAgeHours: 24, + unit: { + mapping: readUnitMappingFrom((file) => readFileSync(path.join(f.directory, file), 'utf8')), + report: first.report!, + context: first.context!, + expectedBuildId: first.context!.buildId, + }, + }); + assert.equal(report.unitEvidence.status, 'matched'); + }); + + for (const mode of ['missing', 'failed', 'changed']) { + it(`surfaces ${mode} execution instead of returning a successful capture`, () => { + const f = fixture(mode); + const result = runUnitTests(f.directory, f.output, 'synthetic-build'); + assert.notEqual(result.exitCode, 0); + assert.ok(result.executionFailure); + if (mode === 'missing') assert.equal(result.report, null); + if (mode === 'changed') assert.equal(result.context?.workingTreeDirty, true); + if (mode === 'failed') assert.equal(result.context?.exitCode, 1); + }); + } +}); diff --git a/scripts/src/ci/assessment/controlAssessmentUnit.ts b/scripts/src/ci/assessment/controlAssessmentUnit.ts new file mode 100644 index 0000000000..5efb284509 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentUnit.ts @@ -0,0 +1,340 @@ +import { createHash } from 'node:crypto'; + +import { hex, object, reportDigest, text, timestamp } from './controlAssessment.ts'; + +import { readMappingContent } from './assessmentConfig.ts'; +import type { AssessmentDefinition } from './assessmentConfig.ts'; + +type Outcome = 'passed' | 'failed' | 'skipped' | 'not-run' | 'unverified' | 'mismatched' | 'stale'; +type Status = 'missing' | 'unverified' | 'matched' | 'mismatched' | 'stale'; +type UnitCase = { + fullName: string; + status: 'passed' | 'failed' | 'skipped'; + reason: string | null; + invocations: number | null; + retryReasons: string[]; + retries: number; + flaky: boolean; +}; + +export type UnitMapping = { digest: string; current: boolean }; +export type UnitContext = { + schemaVersion: 1; + platform: string; + sourceCommit: string; + workingTreeDirty: boolean; + buildId: string; + contractDigest: string; + mappingDigest: string; + reportSha256: string; + startedAt: string; + finishedAt: string; + exitCode: number | null; +}; +export type UnitInput = { + mapping?: UnitMapping; + report?: string; + context?: UnitContext; + executionFailure?: string; + expectedBuildId?: string; +}; + +export function readUnitMappingFrom(read: (file: string) => string, definition: AssessmentDefinition): UnitMapping { + const hash = createHash('sha256'); + for (const file of definition.unit.inputs) hash.update(`${file}\0${read(file).replace(/\r\n/g, '\n')}\0`); + const packagePath = `${definition.packagePath}/package.json`; + const manifest = object(JSON.parse(read(packagePath)), 'Unit package manifest'); + const profile = object(manifest.furn, 'Unit package profile').jestPlatform; + const script = object(manifest.scripts, 'Unit package scripts').test; + hash.update(`${packagePath}\0${JSON.stringify({ profile, script })}\0`); + hash.update(readMappingContent(read, definition)); + const digest = hash.digest('hex'); + return { digest, current: digest === definition.unit.reviewedDigest }; +} + +function count(value: unknown, label: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 0) throw new Error(`${label} must be a nonnegative integer.`); + return Number(value); +} + +export function parseUnitContext(value: unknown): UnitContext { + const data = object(value, 'Unit execution context'); + if ( + data.schemaVersion !== 1 || + !['react', 'win32', 'macos', 'ios', 'android', 'windows'].includes(String(data.platform)) || + typeof data.workingTreeDirty !== 'boolean' + ) { + throw new Error('Unsupported unit context schema, platform, or dirty flag.'); + } + const startedAt = timestamp(data.startedAt, 'Unit execution start'); + const finishedAt = timestamp(data.finishedAt, 'Unit execution finish'); + if (Date.parse(startedAt) > Date.parse(finishedAt)) throw new Error('Unit execution finished before it started.'); + return { + schemaVersion: 1, + platform: text(data.platform, 'Unit platform'), + sourceCommit: hex(data.sourceCommit, 40, 'Unit source commit'), + workingTreeDirty: data.workingTreeDirty, + buildId: text(data.buildId, 'Unit build ID'), + contractDigest: hex(data.contractDigest, 64, 'Unit contract digest'), + mappingDigest: hex(data.mappingDigest, 64, 'Unit mapping digest'), + reportSha256: hex(data.reportSha256, 64, 'Unit report hash'), + startedAt, + finishedAt, + exitCode: data.exitCode === null ? null : count(data.exitCode, 'Unit exit code'), + }; +} + +export function parseComponentJest(content: string, definition: AssessmentDefinition) { + if (Buffer.byteLength(content) > 1_000_000) throw new Error('Unit report exceeds its size limit.'); + const data = object(JSON.parse(content), 'Jest report'); + if (typeof data.success !== 'boolean' || typeof data.wasInterrupted !== 'boolean') throw new Error('Invalid Jest execution flags.'); + if (!Array.isArray(data.testResults) || !data.testResults.length) throw new Error('Jest report must contain test suites.'); + const startTime = count(data.startTime, 'Jest start time'); + const parsed = data.testResults.map((value) => parseJestSuite(value, startTime)); + if (new Set(parsed.map(({ name }) => name)).size !== parsed.length) throw new Error('Duplicate Jest suite identity.'); + const selected = definition.unit.testFiles.map((file) => { + const expected = `${definition.packagePath}/${file}`; + const matches = parsed.filter(({ name }) => name === expected || name.endsWith(`/${expected}`)); + if (matches.length !== 1) + throw new Error('Unit report must contain each configured runtime suite exactly once; type tests cannot substitute.'); + return matches[0]; + }); + const entries = selected.flatMap(({ entries }) => entries); + if (new Set(entries.map(({ fullName }) => fullName)).size !== entries.length) { + throw new Error('Duplicate Jest assertion identity.'); + } + const counts = { passed: 0, failed: 0, skipped: 0 }; + const suites = { passed: 0, failed: 0, skipped: 0 }; + for (const suite of parsed) { + if (suite.status === 'passed' || suite.status === 'focused') suites.passed++; + else if (suite.status === 'failed') suites.failed++; + else suites.skipped++; + for (const status of ['passed', 'failed', 'skipped'] as const) counts[status] += suite.counts[status]; + } + if ( + count(data.numTotalTestSuites, 'Jest total suites') !== parsed.length || + count(data.numPassedTestSuites, 'Jest passed suites') !== suites.passed || + count(data.numFailedTestSuites, 'Jest failed suites') !== suites.failed || + count(data.numPendingTestSuites, 'Jest pending suites') !== suites.skipped || + count(data.numTotalTests, 'Jest total') !== counts.passed + counts.failed + counts.skipped || + count(data.numPassedTests, 'Jest passed') !== counts.passed || + count(data.numFailedTests, 'Jest failed') !== counts.failed || + count(data.numPendingTests, 'Jest pending') + count(data.numTodoTests, 'Jest todo') !== counts.skipped + ) + throw new Error('Jest suite/assertion counts disagree with its summary.'); + const runtimeErrors = count(data.numRuntimeErrorTestSuites, 'Jest runtime errors'); + if (runtimeErrors > suites.failed || (data.success && (counts.failed || runtimeErrors || suites.failed))) { + throw new Error('Jest execution status contradicts its failures.'); + } + const retryCounts = (entries: UnitCase[]) => ({ + retries: entries.reduce((sum, entry) => sum + entry.retries, 0), + flaky: entries.filter(({ flaky }) => flaky).length, + }); + return { + entries, + counts: selected.reduce( + (counts, suite) => ({ + passed: counts.passed + suite.counts.passed, + failed: counts.failed + suite.counts.failed, + skipped: counts.skipped + suite.counts.skipped, + }), + { passed: 0, failed: 0, skipped: 0 }, + ), + packageCounts: counts, + packageSuites: parsed.length, + retryCounts: retryCounts(entries), + packageRetryCounts: retryCounts(parsed.flatMap(({ entries }) => entries)), + startTime, + endTime: Math.max(...parsed.map(({ endTime }) => endTime)), + failed: !data.success || data.wasInterrupted || runtimeErrors > 0 || suites.failed > 0, + }; +} + +function parseJestSuite(value: unknown, startTime: number) { + const suite = object(value, 'Jest suite'); + const name = text(suite.name, 'Jest suite filename').replace(/\\/g, '/'); + const suiteStatus = suite.status; + if (suiteStatus !== 'passed' && suiteStatus !== 'failed' && suiteStatus !== 'skipped' && suiteStatus !== 'focused') { + throw new Error('Unsupported Jest suite status.'); + } + if (!Array.isArray(suite.assertionResults)) throw new Error('Jest suite must contain assertionResults.'); + const entries: UnitCase[] = suite.assertionResults.map((value) => { + const entry = object(value, 'Jest assertion'); + const fullName = text(entry.fullName, 'Jest full name'); + const title = text(entry.title, 'Jest title'); + if ( + !Array.isArray(entry.ancestorTitles) || + !entry.ancestorTitles.every((value) => typeof value === 'string') || + [...entry.ancestorTitles, title].join(' ') !== fullName + ) { + throw new Error('Jest assertion identity is inconsistent.'); + } + if (!['passed', 'failed', 'pending', 'todo'].includes(String(entry.status))) throw new Error('Unsupported Jest assertion status.'); + if (!Array.isArray(entry.failureMessages) || entry.failureMessages.some((message) => typeof message !== 'string')) { + throw new Error('Jest failureMessages must be a string array.'); + } + if (entry.status === 'passed' && entry.failureMessages.length) throw new Error('Passed Jest assertion contains a failure.'); + const invocations = entry.invocations === undefined ? null : count(entry.invocations, 'Jest assertion invocations'); + if ( + entry.retryReasons !== undefined && + (!Array.isArray(entry.retryReasons) || entry.retryReasons.some((reason) => typeof reason !== 'string')) + ) { + throw new Error('Jest retryReasons must be a string array.'); + } + const retryReasons: string[] = entry.retryReasons ?? []; + const retries = Math.max(invocations === null ? 0 : invocations - 1, retryReasons.length); + return { + fullName, + status: entry.status === 'passed' || entry.status === 'failed' ? entry.status : 'skipped', + reason: entry.failureMessages.length ? entry.failureMessages.join('\n').slice(0, 2000) : entry.status === 'todo' ? 'todo' : null, + invocations, + retryReasons, + retries, + flaky: entry.status === 'passed' && retries > 0, + }; + }); + const counts = { + passed: entries.filter(({ status }) => status === 'passed').length, + failed: entries.filter(({ status }) => status === 'failed').length, + skipped: entries.filter(({ status }) => status === 'skipped').length, + }; + if ( + (suiteStatus === 'passed' && counts.failed) || + (suiteStatus === 'focused' && (counts.failed || !counts.skipped)) || + (suiteStatus === 'skipped' && (counts.passed || counts.failed)) + ) { + throw new Error('Jest execution status contradicts its failures.'); + } + const suiteStart = count(suite.startTime, 'Jest suite start time'); + const endTime = count(suite.endTime, 'Jest suite end time'); + const earlyRuntimeFailure = suiteStatus === 'failed' && suiteStart === 0 && endTime === 0 && counts.passed === 0; + if (suite.status !== 'skipped' && !earlyRuntimeFailure && (suiteStart < startTime || endTime < suiteStart)) { + throw new Error('Invalid Jest execution interval.'); + } + return { + name, + status: suiteStatus, + entries, + counts, + startTime, + endTime, + }; +} + +export function assessControlUnits(options: { + definition: AssessmentDefinition; + input?: UnitInput; + sourceCommit: string; + workingTreeDirty: boolean; + contractDigest: string; + contractMappingCurrent: boolean; + now: Date; + maxAgeHours?: number; +}) { + const { input, definition } = options; + const report = input?.report === undefined ? undefined : parseComponentJest(input.report, definition); + if (report && !input?.mapping) throw new Error('Unit report requires the current source mapping fingerprint.'); + const context = input?.context; + if (context && !report) throw new Error('Unit execution context requires a unit report.'); + const hash = input?.report === undefined ? null : reportDigest(input.report); + let status: Status = report ? 'matched' : 'missing'; + const reasons: string[] = []; + if (report) { + if ( + context && + (context.sourceCommit !== options.sourceCommit || + context.platform !== definition.unit.platform || + context.reportSha256 !== hash || + context.contractDigest !== options.contractDigest || + context.mappingDigest !== input!.mapping!.digest || + (input?.expectedBuildId !== undefined && context.buildId !== input.expectedBuildId) || + Date.parse(context.finishedAt) > options.now.getTime() || + report.startTime < Date.parse(context.startedAt) || + (report.endTime !== 0 && report.endTime > Date.parse(context.finishedAt))) + ) { + status = 'mismatched'; + reasons.push('Unit report bytes, source, contract, mapping, build, or execution interval do not match their binding.'); + } else if ( + context && + options.maxAgeHours !== undefined && + options.now.getTime() - Date.parse(context.finishedAt) > options.maxAgeHours * 3_600_000 + ) { + status = 'stale'; + reasons.push('Unit evidence exceeds the explicitly requested maximum age.'); + } else if ( + !context || + options.workingTreeDirty || + context.workingTreeDirty || + !input?.expectedBuildId || + options.maxAgeHours === undefined + ) { + status = 'unverified'; + reasons.push( + 'Unit evidence needs an execution-generated context, expected unit build, age policy, and unchanged clean source inputs.', + ); + } + } + const mappingCurrent = input?.mapping?.current ?? null; + if (mappingCurrent === false) + reasons.push('Unit assertions or mock configuration changed since review; update the reviewed mapping before accepting evidence.'); + const current = mappingCurrent === true && options.contractMappingCurrent; + const runFailed = Boolean(report?.failed || input?.executionFailure || (context && context.exitCode !== 0)); + const runRetried = Boolean(report?.packageRetryCounts.retries); + if (runRetried) reasons.push('Retried or flaky unit execution is not qualifying evidence in this pilot.'); + const checks = definition.unit.cases.map((mapping) => { + const recorded = report?.entries.find(({ fullName }) => fullName === mapping.fullName); + let evidenceOutcome: Outcome = 'not-run'; + if (recorded) { + if (!current || status === 'stale') evidenceOutcome = 'stale'; + else if (status === 'mismatched') evidenceOutcome = 'mismatched'; + else if (status !== 'matched' || (recorded.status === 'passed' && (runFailed || runRetried))) evidenceOutcome = 'unverified'; + else evidenceOutcome = recorded.status; + } + return { + ...mapping, + recordedOutcome: recorded?.status ?? 'not-run', + evidenceOutcome, + reason: recorded?.reason ?? null, + invocations: recorded?.invocations ?? null, + retryReasons: recorded?.retryReasons ?? [], + retries: recorded?.retries ?? 0, + flaky: recorded?.flaky ?? false, + }; + }); + const outcome = checks.some(({ evidenceOutcome }) => evidenceOutcome === 'failed') + ? 'failed' + : status === 'matched' && current && runFailed + ? 'execution-failed' + : report && !current + ? 'stale' + : status !== 'matched' + ? status === 'missing' + ? 'not-run' + : status + : runRetried + ? 'retried' + : report!.counts.passed === 0 + ? 'skipped' + : 'passed'; + return { + status, + reasons, + outcome, + environment: definition.unit.environment, + mappingVersion: definition.unit.mappingVersion, + mappingCurrent, + mappingDigest: input?.mapping?.digest ?? null, + reportSha256: hash, + runFailed, + runRetried, + executionFailure: input?.executionFailure ?? null, + context: context ?? null, + caseCounts: report?.counts ?? { passed: 0, failed: 0, skipped: 0 }, + packageCounts: report?.packageCounts ?? null, + packageSuites: report?.packageSuites ?? null, + retryCounts: report?.retryCounts ?? { retries: 0, flaky: 0 }, + packageRetryCounts: report?.packageRetryCounts ?? null, + checks, + unmappedCases: report?.entries.filter(({ fullName }) => !definition.unit.cases.some((mapping) => mapping.fullName === fullName)) ?? [], + }; +} diff --git a/scripts/src/ci/assessment/controlAssessmentUnitExecution.ts b/scripts/src/ci/assessment/controlAssessmentUnitExecution.ts new file mode 100644 index 0000000000..a09a811733 --- /dev/null +++ b/scripts/src/ci/assessment/controlAssessmentUnitExecution.ts @@ -0,0 +1,187 @@ +import { execFileSync, spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; + +import { PLATFORM_ENV_VAR } from '../../const.mts'; +import { object, readControlContract, reportDigest, text, timestamp } from './controlAssessment.ts'; +import { parseUnitContext, readUnitMappingFrom } from './controlAssessmentUnit.ts'; +import type { UnitContext } from './controlAssessmentUnit.ts'; +import { readPackageAssessments, selectAssessment, unitCaptureDirectory } from './assessmentConfig.ts'; +import type { AssessmentDefinition } from './assessmentConfig.ts'; + +export type UnitExecution = { + schemaVersion: 1; + report: string | null; + context: UnitContext | null; + executionFailure: string | null; + exitCode: number; +}; + +function sourceState(root: string) { + const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const status = execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8' }); + const diff = execFileSync('git', ['diff', '--binary', 'HEAD'], { cwd: root, maxBuffer: 32_000_000 }); + return { sourceCommit, workingTreeDirty: status.length > 0, fingerprint: reportDigest(`${status}\0${diff.toString('base64')}`) }; +} + +function beginUnitExecution(root: string, directory: string, definition: AssessmentDefinition, expectedBuildId?: string) { + const initial = sourceState(root); + const read = (file: string) => readFileSync(path.join(root, file), 'utf8'); + const mapping = readUnitMappingFrom(read, definition); + const contract = readControlContract(root, definition); + const runId = randomUUID(); + const runDirectory = path.join(directory, `unit-${runId}`); + mkdirSync(directory, { recursive: true }); + mkdirSync(runDirectory); + const reportPath = path.join(runDirectory, 'jest.json'); + const startedAt = new Date().toISOString(); + return { + root, + definition, + initial, + mapping, + contract, + runDirectory, + reportPath, + startedAt, + buildId: expectedBuildId ?? `local-unit/${definition.component}/${runId}`, + }; +} + +function finishUnitExecution( + capture: ReturnType, + status: number | null, + launchError?: string, +): UnitExecution & { directory: string } { + const { root, definition, initial, mapping, contract, reportPath, startedAt, runDirectory, buildId } = capture; + const finishedAt = new Date().toISOString(); + const final = sourceState(root); + const current = initial.sourceCommit === final.sourceCommit && initial.fingerprint === final.fingerprint; + const report = existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : null; + const executionFailure = launchError + ? `Component unit command could not start: ${launchError}` + : !current + ? 'Source inputs changed during component unit execution; its results cannot qualify.' + : status !== 0 + ? `Component unit command failed (exit ${status ?? 'none'}).` + : report === null + ? `Component unit command produced no fresh Jest report.` + : null; + const context = + report === null + ? null + : parseUnitContext({ + schemaVersion: 1, + platform: definition.unit.platform, + sourceCommit: initial.sourceCommit, + workingTreeDirty: initial.workingTreeDirty || final.workingTreeDirty || !current, + buildId, + contractDigest: contract.digest, + mappingDigest: mapping.digest, + reportSha256: reportDigest(report), + startedAt, + finishedAt, + exitCode: status, + }); + const execution: UnitExecution = { + schemaVersion: 1, + report, + context, + executionFailure, + exitCode: executionFailure ? status || 1 : 0, + }; + writeFileSync(path.join(runDirectory, 'execution.json'), `${JSON.stringify(execution, null, 2)}\n`); + if (context) writeFileSync(path.join(runDirectory, 'context.json'), `${JSON.stringify(context, null, 2)}\n`); + return { ...execution, directory: runDirectory }; +} + +export function runUnitTests( + root: string, + directory: string, + expectedBuildId?: string, + definition = selectAssessment(root), +): UnitExecution & { directory: string } { + const manifest = object(JSON.parse(readFileSync(path.join(root, 'package.json'), 'utf8')), 'Root manifest'); + const version = /^yarn@(\d+\.\d+\.\d+)$/.exec(text(manifest.packageManager, 'Package manager'))?.[1]; + if (!version) throw new Error('Unit collection requires the repository-pinned Yarn release.'); + const yarn = path.join(root, '.yarn', 'releases', `yarn-${version}.cjs`); + if (!existsSync(yarn)) throw new Error(`Pinned Yarn entry point is missing: ${yarn}.`); + const capture = beginUnitExecution(root, directory, definition, expectedBuildId); + const env: NodeJS.ProcessEnv = { ...process.env, [PLATFORM_ENV_VAR]: definition.unit.platform }; + delete env.FURN_ASSESSMENT_UNIT_CAPTURE; + const result = spawnSync( + process.execPath, + [ + yarn, + 'workspace', + definition.packageName, + 'test', + ...definition.unit.testFiles, + '--runInBand', + '--json', + '--outputFile', + capture.reportPath, + ], + { cwd: root, env, stdio: 'inherit' }, + ); + return finishUnitExecution(capture, result.status, result.error?.message); +} + +/** Decorates the existing package invocation; the supplied runner is called exactly once. */ +export async function runJestWithUnitCapture( + packageRoot: string, + platform: string, + args: string[], + run: (args: string[]) => Promise, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const directory = env.FURN_ASSESSMENT_UNIT_CAPTURE; + if (!directory) return run(args); + const root = execFileSync('git', ['rev-parse', '--show-toplevel'], { cwd: packageRoot, encoding: 'utf8' }).trim(); + const definitions = readPackageAssessments(realpathSync.native(root), realpathSync.native(packageRoot)).filter( + (definition) => definition.unit.platform === platform, + ); + if (!definitions.length) return run(args); + if (env[PLATFORM_ENV_VAR] !== undefined && env[PLATFORM_ENV_VAR] !== platform) + throw new Error('Unit capture requires the declared package Jest mock.'); + if (args.some((arg) => arg === '--outputFile' || arg.startsWith('--outputFile=') || arg === '--listTests')) { + throw new Error('Assessment capture cannot replace explicit output/list-only Jest options.'); + } + const destination = path.resolve(directory); + const start = object(JSON.parse(readFileSync(path.join(destination, 'unit-collection-start.json'), 'utf8')), 'Unit collection start'); + if (start.schemaVersion !== 1) throw new Error('Unsupported unit collection capture schema.'); + const identity = object(start.identity, 'Unit collection identity'); + const began = timestamp(start.startedAt, 'Unit collection start time'); + const captures = definitions.map((definition) => { + const output = unitCaptureDirectory(destination, definition); + if (existsSync(path.join(output, 'unit-execution.json'))) + throw new Error('A unit execution was already collected; refusing replacement.'); + const capture = beginUnitExecution(root, output, definition, text(start.buildId, 'Unit collection build ID')); + if (capture.initial.sourceCommit !== identity.testedCommit || Date.parse(capture.startedAt) < Date.parse(began)) + throw new Error('Existing unit invocation does not match pre-execution source or interval.'); + return capture; + }); + const reportPath = captures[0].reportPath; + let exitCode: number | null = null; + try { + exitCode = await run([...args, '--json', '--outputFile', reportPath]); + return exitCode; + } finally { + for (const capture of captures) { + const execution = finishUnitExecution({ ...capture, reportPath }, exitCode); + writeFileSync( + path.join(unitCaptureDirectory(destination, capture.definition), 'unit-execution.json'), + `${JSON.stringify( + { + ...execution, + identity, + collectionStartedAt: began, + }, + null, + 2, + )}\n`, + ); + } + } +} diff --git a/scripts/src/ci/controlAssessmentCiTask.ts b/scripts/src/ci/controlAssessmentCiTask.ts new file mode 100644 index 0000000000..9934ba666a --- /dev/null +++ b/scripts/src/ci/controlAssessmentCiTask.ts @@ -0,0 +1,142 @@ +import { appendFileSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { object, text } from './assessment/controlAssessment.ts'; +import { + assessmentArtifactName, + beginCiExecution, + beginCiUnitCollection, + combineCiEvidence, + finishCiExecution, + git, +} from './assessment/controlAssessmentCi.ts'; +import { + AssessmentGitHub, + inspectReportUpdates, + preparePublications, + publishAssessments, + readBundles, + verifyReportUpdates, +} from './assessment/controlAssessmentGithub.ts'; +import type { Candidates } from './assessment/controlAssessmentGithub.ts'; +import { discoverAssessments } from './assessment/assessmentConfig.ts'; + +function output(name: string, value: string | boolean): void { + if (/[\r\n]/.test(String(value))) throw new Error('Invalid multiline workflow output.'); + appendFileSync(text(process.env.GITHUB_OUTPUT, 'GITHUB_OUTPUT'), `${name}=${value}\n`); +} + +function notice(message: string): void { + process.stdout.write(`${message}\n`); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n\n`); +} + +async function main(args = process.argv.slice(2)): Promise { + const env = process.env; + const root = git(process.cwd(), ['rev-parse', '--show-toplevel']); + const command = args[0]; + if (['begin', 'finish', 'combine'].includes(command) && !discoverAssessments(root).length) + throw new Error('No component assessment is configured.'); + if (command === 'begin') { + for (const definition of discoverAssessments(root)) beginCiExecution(root, env, new Date(), definition); + notice('Captured configured component inputs. Hosted physical input remains disabled.'); + } else if (command === 'begin-unit') { + beginCiUnitCollection(root, env); + notice('Prepared capture of the existing component Jest invocation; no additional unit tests are launched.'); + } else if (command === 'combine') { + for (const definition of discoverAssessments(root)) { + const bundle = combineCiEvidence(root, env, new Date(), definition); + notice( + `Assessed ${definition.component} from run ${bundle.identity.runId}, attempt ${bundle.identity.runAttempt}; no tests were rerun.`, + ); + } + } else if (command === 'finish') { + for (const definition of discoverAssessments(root)) { + const bundle = finishCiExecution(root, env, new Date(), definition); + notice( + `Advisory ${definition.component} assessment generated for run ${bundle.identity.runId}, attempt ${bundle.identity.runAttempt}. No maturity stage assigned.`, + ); + } + } else if (command === 'inspect') { + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const candidate = await inspectReportUpdates(root, env, github); + output('candidate', candidate.reportOnly); + if (candidate.reportOnly) { + const candidatePath = path.join(text(env.RUNNER_TEMP, 'RUNNER_TEMP'), 'component-assessment-candidate.json'); + writeFileSync(candidatePath, JSON.stringify(candidate)); + output('candidate-path', candidatePath); + output('run-id', candidate.candidates[0].reference.runId); + output('artifact-name', `${assessmentArtifactName}-${candidate.candidates[0].reference.runAttempt}`); + notice('Report-only candidate found. Source jobs remain gated until the original evidence and rendered report are verified.'); + } else { + output('report-only', false); + notice(candidate.reason); + } + } else if (command === 'verify') { + const candidate = JSON.parse(readFileSync(text(env.CANDIDATE_PATH, 'CANDIDATE_PATH'), 'utf8')) as Candidates; + verifyReportUpdates(root, candidate, readBundles(root, text(env.BUNDLE_DIRECTORY, 'BUNDLE_DIRECTORY'))); + if (!candidate.reportOnly) throw new Error('No report-only candidate.'); + output('report-only', true); + output('source-result', candidate.candidates[0].sourceResult); + notice( + `Verified unchanged tested inputs and exact generated report/SPEC assessment fields for every configured control. Reusing source validation result: ${candidate.candidates[0].sourceResult}.`, + ); + } else if (command === 'find-artifact') { + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const found = await github.hasArtifact(text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), String(run.id), Number(run.run_attempt)); + output('found', found); + output('artifact-name', `${assessmentArtifactName}-${Number(run.run_attempt)}`); + notice( + found + ? 'Found the source attempt assessment bundle.' + : 'No assessment artifact exists for this attempt. Report-only verification runs do not execute or republish native assessments.', + ); + } else if (command === 'prepare-publication') { + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const github = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const publications = await preparePublications(root, readBundles(root, text(env.BUNDLE_DIRECTORY, 'BUNDLE_DIRECTORY')), github, { + repository: text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), + runId: String(run.id), + runAttempt: Number(run.run_attempt), + defaultBranch: text(object(event.repository, 'Repository').default_branch, 'Default branch'), + }); + output('eligible', publications.length > 0 && publications.every((publication) => publication.eligible)); + if (publications.length > 0 && publications.every((publication) => publication.eligible)) { + notice( + 'Validated every configured component, the source attempt, native evidence, merged input tree, target PR, and allowlisted assessment write paths.', + ); + } else { + for (const publication of publications) if (!publication.eligible) notice(publication.reason); + } + } else if (command === 'publish') { + // Revalidate after acquiring the write credential; never trust a stale prepared mutation. + const event = object(JSON.parse(readFileSync(text(env.GITHUB_EVENT_PATH, 'GITHUB_EVENT_PATH'), 'utf8')), 'Workflow event'); + const run = object(event.workflow_run, 'Source workflow run'); + const readGitHub = new AssessmentGitHub({ token: text(env.GITHUB_TOKEN, 'GITHUB_TOKEN') }); + const publications = await preparePublications(root, readBundles(root, text(env.BUNDLE_DIRECTORY, 'BUNDLE_DIRECTORY')), readGitHub, { + repository: text(env.GITHUB_REPOSITORY, 'GITHUB_REPOSITORY'), + runId: String(run.id), + runAttempt: Number(run.run_attempt), + defaultBranch: text(object(event.repository, 'Repository').default_branch, 'Default branch'), + }); + if (!publications.length || publications.some((publication) => !publication.eligible)) { + for (const publication of publications) if (!publication.eligible) notice(publication.reason); + return; + } + const writer = new AssessmentGitHub({ token: text(env.ASSESSMENT_WRITE_TOKEN, 'ASSESSMENT_WRITE_TOKEN') }); + const commit = await publishAssessments(publications, writer); + notice( + `Published the generated component summary and SPEC assessment fields in commit ${commit}. Contract content, code, dependencies and workflows are unchanged.`, + ); + } else { + throw new Error('Expected begin, begin-unit, finish, combine, inspect, verify, find-artifact, prepare-publication, or publish.'); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); +} diff --git a/scripts/src/cli.ts b/scripts/src/cli.ts index 2d0b17c244..e78fcf5f38 100644 --- a/scripts/src/cli.ts +++ b/scripts/src/cli.ts @@ -8,6 +8,7 @@ import { JestCommand } from './tasks/jest.ts'; import { CheckPublishingCommand } from './tasks/checkPublishingTask.ts'; import { CheckAgentInstructionsCommand } from './tasks/checkAgentInstructionsTask.ts'; import { CheckPackageContentsCommand } from './tasks/checkPackageContentsTask.ts'; +import { AssessControlCommand } from './tasks/assessControlTask.ts'; const cli = new Cli({ binaryLabel: 'fluentui-scripts CLI', @@ -22,6 +23,7 @@ cli.register(JestCommand); cli.register(CheckPublishingCommand); cli.register(CheckAgentInstructionsCommand); cli.register(CheckPackageContentsCommand); +cli.register(AssessControlCommand); cli.register(Builtins.DefinitionsCommand); cli.register(Builtins.HelpCommand); diff --git a/scripts/src/pkgContext.ts b/scripts/src/pkgContext.ts index 90dddb89d7..9ed4cb5c3a 100644 --- a/scripts/src/pkgContext.ts +++ b/scripts/src/pkgContext.ts @@ -13,6 +13,7 @@ export type PlatformTarget = 'react' | 'win32' | 'macos' | 'ios' | 'android' | ' export type FurnConfig = { packageType?: PackageType; jestPlatform?: PlatformTarget; + assessments?: string[]; knip?: { entry?: string[]; extraEntries?: string[]; diff --git a/scripts/src/tasks/assessControlTask.ts b/scripts/src/tasks/assessControlTask.ts new file mode 100644 index 0000000000..99c63c878f --- /dev/null +++ b/scripts/src/tasks/assessControlTask.ts @@ -0,0 +1,261 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { Command, Option } from 'clipanion'; + +import { + assessControl, + formatAssessmentMarkdown, + object, + parseEvidenceContext, + parseNativeReport, + readControlContract, + readNativeEvidence, + reportDigest, + text, +} from '../ci/assessment/controlAssessment.ts'; +import { specAssessmentFor, updateSpecAssessment } from '../ci/assessment/controlAssessmentSpec.ts'; +import { parseUnitContext, readUnitMappingFrom } from '../ci/assessment/controlAssessmentUnit.ts'; +import { runUnitTests } from '../ci/assessment/controlAssessmentUnitExecution.ts'; +import { discoverAssessments, unconfiguredControls } from '../ci/assessment/assessmentConfig.ts'; +import type { AssessmentDefinition } from '../ci/assessment/assessmentConfig.ts'; +import { combineCiEvidence, git } from '../ci/assessment/controlAssessmentCi.ts'; +import { formatAssessmentDocument } from '../ci/assessment/assessmentMarkdown.ts'; + +function sameFile(first: string, second: string): boolean { + if (path.relative(first, second) === '') return true; + if (!existsSync(first) || !existsSync(second)) return false; + const firstInfo = statSync(first, { bigint: true }); + const secondInfo = statSync(second, { bigint: true }); + return firstInfo.dev === secondInfo.dev && firstInfo.ino === secondInfo.ino; +} + +export class AssessControlCommand extends Command { + static override paths = [['assess-components'], ['assess-control']]; + + static override usage = Command.Usage({ + description: 'Assess configured package components from existing native and JavaScript unit evidence', + details: + 'Discovers package-owned assessment mappings and reads existing reports. --run-unit-tests explicitly runs configured runtime tests locally; CI only combines existing evidence. Unconfigured controls are listed, never treated as passes. No maturity stage is assigned. See @fluentui-react-native/components/references/assessments/README.md.', + examples: [ + [ + 'Assess an existing native report', + '$0 assess-components --component checkbox --evidence apps/storybook/artifacts/desktop//run.json', + ], + ], + }); + + component = Option.String('--component', { description: 'Select one configured component; otherwise assess all configured controls' }); + packageDirectory = Option.String('--package', { description: 'Owning package directory; defaults to the current package' }); + list = Option.Boolean('--list', false, { description: 'List configured and unconfigured package controls without assessing them' }); + ci = Option.Boolean('--ci', false, { description: 'Combine existing CI evidence without running tests or editing tracked files' }); + evidence = Option.String('--evidence', { + description: 'Canonical desktop-test run.json with its Storybook companion, or a historical WDIO/smoke report', + }); + contextFile = Option.String('--context', { + description: 'Optional execution-generated provenance sidecar, never synthesized from the current checkout', + }); + unitEvidence = Option.String('--unit-evidence', { description: 'Existing Jest JSON; only configured runtime suites are mapped' }); + unitContextFile = Option.String('--unit-context', { description: 'Execution-generated unit provenance sidecar' }); + unitBuildId = Option.String('--unit-build-id', { description: 'Expected unit build ID for replayed evidence' }); + runUnitTests = Option.Boolean('--run-unit-tests', false, { + description: 'Explicitly run the configured runtime tests locally and capture fresh evidence', + }); + buildId = Option.String('--build-id', { description: 'Expected test build ID; required to accept bound evidence' }); + targetId = Option.String('--target-id', { description: 'Expected native target instance; required for bound desktop-test evidence' }); + maxAgeHours = Option.String('--max-age-hours', { description: 'Explicit positive evidence-age limit; no default readiness policy' }); + output = Option.String('--output', { description: 'Report directory; defaults to apps/storybook/artifacts/win32/assessment' }); + updateSpec = Option.Boolean('--update-spec', false, { + description: 'Also write the generated SPEC assessment snapshot and local report', + }); + + async execute() { + const root = git(process.cwd(), ['rev-parse', '--show-toplevel']); + const packageRoot = + this.packageDirectory === undefined + ? this.path[0] === 'assess-components' + ? process.cwd() + : undefined + : path.resolve(this.packageDirectory); + const configured = discoverAssessments(root, packageRoot); + const definitions = configured.filter((definition) => this.component === undefined || definition.component === this.component); + if (!definitions.length) throw new Error('No approved assessment mapping is configured for the requested component/package.'); + if (this.component !== undefined && definitions.length !== 1) + throw new Error('Component selection is ambiguous; select its package explicitly.'); + const packages = [...new Set(configured.map(({ packagePath }) => packagePath))]; + const inventory = { + configured: definitions.map(({ component, packagePath, mappingPath }) => ({ component, packagePath, mappingPath })), + unconfigured: packages.flatMap((packagePath) => + unconfiguredControls(path.join(root, packagePath), configured).map((component) => ({ + component, + packagePath, + status: 'not-configured', + })), + ), + }; + if (this.list) { + this.context.stdout.write(`${JSON.stringify(inventory, null, 2)}\n`); + return 0; + } + if (inventory.unconfigured.length) + this.context.stdout.write( + `Not configured for assessment: ${inventory.unconfigured.map(({ component }) => component).join(', ')}. These are not passes.\n`, + ); + if (this.ci || process.env.FURN_ASSESSMENT_CI === 'true') { + if ( + this.runUnitTests || + this.updateSpec || + this.evidence || + this.unitEvidence || + this.contextFile || + this.unitContextFile || + this.output || + this.buildId || + this.unitBuildId || + this.targetId || + this.maxAgeHours + ) + throw new Error( + 'CI assessment consumes only the captured workflow evidence; local execution, replay and tracked writes are not allowed.', + ); + for (const definition of definitions) combineCiEvidence(root, process.env, new Date(), definition); + this.context.stdout.write( + `Assessed ${definitions.length} configured component(s) using existing CI evidence; no tests were rerun.\n`, + ); + return 0; + } + let exitCode = 0; + for (const definition of definitions) exitCode = Math.max(exitCode, await this.assess(root, definition, definitions.length > 1)); + return exitCode; + } + + private async assess(root: string, definition: AssessmentDefinition, multiple: boolean) { + if (this.runUnitTests && (this.unitEvidence !== undefined || this.unitContextFile !== undefined || this.unitBuildId !== undefined)) { + throw new Error('--run-unit-tests cannot be combined with replayed unit evidence, context, or build ID.'); + } + if (this.unitContextFile !== undefined && this.unitEvidence === undefined) throw new Error('--unit-context requires --unit-evidence.'); + if (this.unitBuildId !== undefined && this.unitEvidence === undefined) throw new Error('--unit-build-id requires --unit-evidence.'); + const assessmentSpecPath = definition.specPath; + const assessmentPackagePath = `${definition.packagePath}/package.json`; + const unitRun = this.runUnitTests + ? runUnitTests(root, path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment-units'), undefined, definition) + : undefined; + const sourceCommit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const workingTreeDirty = + execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8' }).length > 0; + const input = this.evidence === undefined ? undefined : path.resolve(this.evidence); + const contextFile = this.contextFile === undefined ? undefined : path.resolve(this.contextFile); + const unitInput = unitRun?.report + ? path.join(unitRun.directory, 'jest.json') + : this.unitEvidence === undefined + ? undefined + : path.resolve(this.unitEvidence); + const unitContextFile = unitRun?.context + ? path.join(unitRun.directory, 'context.json') + : this.unitContextFile === undefined + ? undefined + : path.resolve(this.unitContextFile); + const content = input === undefined ? undefined : readNativeEvidence(input); + const specPath = path.join(root, assessmentSpecPath); + const originalSpec = readFileSync(specPath, 'utf8'); + const sourceMappingPath = path.join(root, definition.mappingPath); + const originalMapping = readFileSync(sourceMappingPath, 'utf8'); + const report = assessControl({ + contract: readControlContract(root, definition), + sourceCommit, + workingTreeDirty, + now: new Date(), + report: content === undefined ? undefined : parseNativeReport(JSON.parse(content)), + reportSha256: content === undefined ? undefined : reportDigest(content), + context: contextFile === undefined ? undefined : parseEvidenceContext(JSON.parse(readFileSync(contextFile, 'utf8'))), + expectedBuildId: this.buildId, + expectedTargetId: this.targetId, + maxAgeHours: this.maxAgeHours === undefined ? undefined : Number(this.maxAgeHours), + unit: + unitRun || unitInput !== undefined + ? { + mapping: readUnitMappingFrom((file) => readFileSync(path.join(root, file), 'utf8'), definition), + report: unitRun?.report ?? (unitInput === undefined ? undefined : readFileSync(unitInput, 'utf8')), + context: + unitRun?.context ?? + (unitContextFile === undefined ? undefined : parseUnitContext(JSON.parse(readFileSync(unitContextFile, 'utf8')))), + executionFailure: unitRun?.executionFailure ?? undefined, + expectedBuildId: unitRun?.context?.buildId ?? this.unitBuildId, + } + : undefined, + }); + const baseDirectory = + this.output === undefined ? path.join(root, 'apps', 'storybook', 'artifacts', 'win32', 'assessment') : path.resolve(this.output); + const directory = multiple ? path.join(baseDirectory, definition.packagePath, definition.component) : baseDirectory; + const jsonPath = path.join(directory, 'assessment.json'); + const markdownPath = path.join(directory, 'assessment.md'); + const mappingPath = path.join(directory, 'assessment.mapping.json'); + const localSummary = path.join(path.dirname(specPath), 'assessment', 'ASSESSMENT.md'); + const outputs = this.updateSpec ? [jsonPath, markdownPath, mappingPath, specPath, localSummary] : [jsonPath, markdownPath, mappingPath]; + for (const output of outputs) { + if ([input, contextFile, unitInput, unitContextFile].some((file) => file !== undefined && sameFile(file, output))) { + throw new Error('Assessment output must not overwrite an input report or context.'); + } + } + if (sameFile(jsonPath, markdownPath)) throw new Error('Assessment JSON and Markdown must be separate files.'); + for (const output of [jsonPath, markdownPath, mappingPath, ...(this.updateSpec ? [localSummary] : [])]) { + if (sameFile(output, specPath)) throw new Error('Assessment output must not overwrite the specification.'); + if (sameFile(output, sourceMappingPath)) throw new Error('Assessment output must not overwrite the reviewed source mapping.'); + } + if (this.updateSpec && [jsonPath, markdownPath].some((output) => sameFile(output, localSummary))) { + throw new Error('Local summary must be separate from the artifact reports.'); + } + let updatedSpec: string | undefined; + if (this.updateSpec) { + for (const file of [specPath, localSummary]) { + if (existsSync(file) && (!lstatSync(file).isFile() || lstatSync(file).nlink !== 1)) { + throw new Error('SPEC snapshot destinations must be regular, unlinked files.'); + } + } + const manifest = object(JSON.parse(readFileSync(path.join(root, assessmentPackagePath), 'utf8')), 'Component package manifest'); + updatedSpec = updateSpecAssessment(originalSpec, specAssessmentFor(report, text(manifest.version, 'Component package version'))); + mkdirSync(path.dirname(localSummary), { recursive: true }); + if (path.relative(realpathSync(path.dirname(specPath)), realpathSync(path.dirname(localSummary))) !== 'assessment') { + throw new Error('Local assessment directory must not redirect outside its component.'); + } + } + mkdirSync(directory, { recursive: true }); + if (readFileSync(sourceMappingPath, 'utf8') !== originalMapping) + throw new Error('Mapping changed during assessment; refusing an inconsistent artifact.'); + writeFileSync(mappingPath, originalMapping); + writeFileSync( + jsonPath, + `${JSON.stringify( + { + ...report, + inputs: { + report: input ?? null, + context: contextFile ?? null, + unitReport: unitInput ?? null, + unitContext: unitContextFile ?? null, + unitExecuted: this.runUnitTests, + }, + }, + null, + 2, + )}\n`, + ); + writeFileSync(markdownPath, formatAssessmentDocument(formatAssessmentMarkdown(report), definition.summaryPath)); + if (updatedSpec !== undefined) { + if (readFileSync(specPath, 'utf8') !== originalSpec) + throw new Error('SPEC changed during assessment; refusing concurrent overwrite.'); + writeFileSync( + localSummary, + formatAssessmentDocument( + `\n\n${formatAssessmentMarkdown(report)}`, + definition.summaryPath, + ), + ); + writeFileSync(specPath, updatedSpec); + } + this.context.stdout.write( + `Assessment report generated: ${report.assessmentOutcome}; readiness not established (advisory).\n${jsonPath}\n${markdownPath}\n${mappingPath}\n`, + ); + return unitRun?.exitCode ?? 0; + } +} diff --git a/scripts/src/tasks/jest.ts b/scripts/src/tasks/jest.ts index 9acd468616..471579faf0 100644 --- a/scripts/src/tasks/jest.ts +++ b/scripts/src/tasks/jest.ts @@ -8,6 +8,7 @@ import { hasJestConfig } from '../preinstall/tool-versions.ts'; import { PackageContext, type PlatformTarget } from '../pkgContext.ts'; import { PLATFORM_ENV_VAR, DEFULT_WIN_PLATFORM } from '../const.mts'; import { getFileMetadata } from '../utils/fileData.ts'; +import { runJestWithUnitCapture } from '../ci/assessment/controlAssessmentUnitExecution.ts'; import { buildJestOptionTable, matchTestFiles, @@ -103,7 +104,9 @@ export class JestCommand extends Command { if (files && files.length > 0) { console.log(`Running jest for platform ${platform} with ${files.length} test files.`); const options = getPlatformOptions(platform, defaultPlatform); - const result = await runScript('jest', [...args, ...parsed.passthrough, ...files], options); + const result = await runJestWithUnitCapture(context.root, platform, [...args, ...parsed.passthrough, ...files], (jestArgs) => + runScript('jest', jestArgs, options), + ); if (result !== 0) { return result; }