From 958148c41791fa359d96383199c4a46e776ec2c7 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Tue, 20 Oct 2026 17:18:05 -0700 Subject: [PATCH 1/2] Add Codex as an advisory red-team audit engine `scripts/security-audit-local.sh --engine codex` runs the same audit prompts on gpt-6-sol under the daybreak_blue Cyber access program, sandboxed to the working tree. Its canary takes a CI canary run's seed key (`--key `) and that run's decrypted scorecard to count the seeds either engine caught, then prints the publishing command. `.github/workflows/codex-audit-report.yaml` publishes from non-secret dispatch inputs only (commit, model, verdicts, canary counts). The public-body builder checks each input, the commit against main, the dispatcher's admin role, and the counts against the CI canary run's own record, then comments on the status issue and uploads `codex-recall`. The recall graph on /security draws three lines: Claude, Codex, and caught by either. security-audit.md keeps S8's Claude split as the release gate and adds a "Red team" section with its cadence (quarterly, plus before releases that change remote control or Hosted) and S275. Co-Authored-By: Claude Opus 5.5 --- .github/audit/ci-and-secrets.md | 2 +- .github/workflows/codex-audit-report.yaml | 113 ++++++++++++ docs/specs/deploy.md | 2 + docs/specs/security-audit.md | 27 ++- docs/specs/security-audit.rationale.md | 10 ++ docs/specs/security-ci.md | 2 +- docs/specs/security.md | 2 + scripts/security-audit-canary.mjs | 19 +- scripts/security-audit-canary.test.mjs | 44 ++++- scripts/security-audit-local.sh | 141 ++++++++++++--- scripts/security-audit-public-body.mjs | 149 +++++++++++++++- scripts/security-audit.test.mjs | 133 ++++++++++++++ scripts/security-rule-ids.json | 2 +- scripts/spec-word-budgets.json | 6 +- scripts/workflow-lint.mjs | 3 +- scripts/workflow-lint.test.mjs | 2 + website/scripts/generate-canary-recall.js | 98 ++++++++-- .../scripts/generate-canary-recall.test.js | 50 +++++- website/src/components/CanaryRecall.test.tsx | 51 +++++- website/src/components/CanaryRecall.tsx | 167 +++++++++++++++--- 20 files changed, 946 insertions(+), 77 deletions(-) create mode 100644 .github/workflows/codex-audit-report.yaml diff --git a/.github/audit/ci-and-secrets.md b/.github/audit/ci-and-secrets.md index 88dc138e6..5c9622208 100644 --- a/.github/audit/ci-and-secrets.md +++ b/.github/audit/ci-and-secrets.md @@ -30,7 +30,7 @@ Delegate only by these streams (`.github/audit/_preamble.md` -> "Work streams"). - `actions` — `docs/specs/security-ci.md`: "GitHub Actions Policies", "Automated Maintainer (tend)". - `releases` — `docs/specs/security-ci.md`: "Hosted Deployments", "VS Code Extension Releases", "Desktop Releases". -- `audit` — `docs/specs/security-audit.md`: "Schedule and gate", "Deterministic checks", "Skipping an unchanged audit", "Domains", "Orchestration", "Outcomes and reporting", "Findings", "Embargo", "Findings ledger", "Disclosure", "Environment and `AUDIT_PAT`", "Canary recall". +- `audit` — `docs/specs/security-audit.md`: "Schedule and gate", "Deterministic checks", "Skipping an unchanged audit", "Domains", "Orchestration", "Outcomes and reporting", "Findings", "Embargo", "Findings ledger", "Disclosure", "Environment and `AUDIT_PAT`", "Canary recall", "Red team". - `qualitative` — the qualitative pass below, once, over the whole of this domain's scope. ## Qualitative pass diff --git a/.github/workflows/codex-audit-report.yaml b/.github/workflows/codex-audit-report.yaml new file mode 100644 index 000000000..a46df8d9b --- /dev/null +++ b/.github/workflows/codex-audit-report.yaml @@ -0,0 +1,113 @@ +name: codex-audit-report + +# Publishes the red team's report (docs/specs/security-audit.md -> "Red team"): +# what a maintainer's `scripts/security-audit-local.sh --engine codex` run +# computed, typed in as dispatch inputs. Nothing here runs a model or reads a +# secret, and nothing here gates a release: release.yml waits on +# security-audit.yaml alone. The inputs are non-secret by construction — +# computed verdicts and counts, never a finding — and +# `scripts/security-audit-public-body.mjs` checks each against its pattern, +# and the counts against the CI canary run they share seeds with, before it +# posts anything. +on: + workflow_dispatch: + inputs: + commit: + description: "The commit on main the Codex run audited (40-hex)" + type: string + required: true + model: + description: "The Codex model the run used" + type: string + required: true + default: gpt-6-sol + verdicts: + description: "Each domain's computed verdict: `domain=PASS|FAIL|INCONCLUSIVE …`; empty for a canary alone" + type: string + default: "" + canary_run: + description: "The CI canary run whose key seeded the Codex canary; empty for verdicts alone" + type: string + default: "" + canary_seeded: + description: "Seeds applied" + type: string + default: "" + canary_caught: + description: "Seeds Codex caught" + type: string + default: "" + canary_either: + description: "Seeds Claude or Codex caught" + type: string + default: "" + canary_unfinished: + description: "Codex domains that left no finished report" + type: string + default: "" + +run-name: codex-audit-report (advisory, not an audit) + +# Each job declares its own. +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 10 + # `issues: write` for the one comment on the status issue; `actions: read` + # to read the CI canary run and download its `canary-recall`. + permissions: + contents: read + issues: write + actions: read + steps: + - name: Refuse any ref but main + run: | + [ "$GITHUB_REF" = refs/heads/main ] && exit 0 + echo "::error::A red-team report publishes from main only." + exit 1 + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: package.json + + # Inputs reach the builder through the environment, never interpolated + # into the script. It refuses a dispatch by anyone but an admin. + - name: Publish the red-team report + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT: ${{ inputs.commit }} + MODEL: ${{ inputs.model }} + VERDICTS: ${{ inputs.verdicts }} + CANARY_RUN: ${{ inputs.canary_run }} + CANARY_SEEDED: ${{ inputs.canary_seeded }} + CANARY_CAUGHT: ${{ inputs.canary_caught }} + CANARY_EITHER: ${{ inputs.canary_either }} + CANARY_UNFINISHED: ${{ inputs.canary_unfinished }} + run: | + mkdir -p "$RUNNER_TEMP/codex-recall" + node scripts/security-audit-public-body.mjs --engine codex \ + --date "$(date -u +%Y-%m-%dT%H:%MZ)" --run-url "$RUN_URL" \ + --commit "$COMMIT" --model "$MODEL" --verdicts "$VERDICTS" \ + --canary-run "$CANARY_RUN" --canary-seeded "$CANARY_SEEDED" \ + --canary-caught "$CANARY_CAUGHT" --canary-either "$CANARY_EITHER" \ + --canary-unfinished "$CANARY_UNFINISHED" \ + --record "$RUNNER_TEMP/codex-recall/codex-recall.json" + + # The counts website/scripts/generate-canary-recall.js reads beside the + # CI canary run's own `canary-recall`. + - name: Upload the red-team recall + if: inputs.canary_run != '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: codex-recall + path: ${{ runner.temp }}/codex-recall/codex-recall.json + retention-days: 90 + if-no-files-found: error diff --git a/docs/specs/deploy.md b/docs/specs/deploy.md index 6b37b0f8a..27e906d5f 100644 --- a/docs/specs/deploy.md +++ b/docs/specs/deploy.md @@ -21,6 +21,8 @@ Source of truth: `create_release` in `scripts/sign-and-deploy.sh`; `publish-vsco **Must ship a desktop release that carries, or changes, the one-time link version only after Hosted production serves that version** — its rendezvous routes and its `/connect/` page (`docs/specs/one-time.md` -> "Link"): run `.github/workflows/hosted-production.yml` first and confirm its live verification passed, `oneTimeSmoke` included. **Hosted must never stop serving a link version a shipped Burrow emits.** +A release that changes remote control or Hosted runs the red team first (`docs/specs/security-audit.md` -> "Red team"). + Human-driven, in order: 1. **Update website snapshots** — run `node website/scripts/generate-deps.js` and `node website/scripts/generate-canary-recall.js` (on your `gh` login), review the diffs in `website/src/data/dependencies-{npm,cargo,runtime}.json` and `website/src/data/canary-recall.json`, commit if changed. diff --git a/docs/specs/security-audit.md b/docs/specs/security-audit.md index a38a1d50c..6d43119f2 100644 --- a/docs/specs/security-audit.md +++ b/docs/specs/security-audit.md @@ -1,6 +1,6 @@ # Security Audit -> - Owns how the security specs are audited: the schedule and the release gate, the deterministic checks and the skip they allow, the domains and their prompts, the orchestration contract, the three outcomes, the reporting steps, the embargo on finding detail and its disclosure, the environment that holds `AUDIT_PAT`, and the canary that measures recall. +> - Owns how the security specs are audited: the schedule and the release gate, the deterministic checks and the skip they allow, the domains and their prompts, the orchestration contract, the three outcomes, the reporting steps, the embargo on finding detail and its disclosure, the environment that holds `AUDIT_PAT`, the canary that measures recall, and the advisory Codex red team. > - Defers what is audited to `docs/specs/security.md` and the specs it names, and each agent's procedure to its prompt in `.github/audit/`. > - Read `docs/specs/security.md` first. @@ -48,7 +48,7 @@ CI and the local runner share the prompts and their scopes in `.github/audit/` ( - **FAIL IF** `S5` a `docs/specs/security*.md` spec (not a `.rationale.md`) is in no domain's scope, or in two, or a scope names a file that does not exist (rationale). - **FAIL IF** `S6` `.github/audit/orchestrator.md` lets the orchestrator audit anything itself (rationale). - **FAIL IF** `S7` the audit stops fanning out to a dedicated `remote` subagent scoped to `docs/specs/security-remote.md`, a dedicated `local` subagent scoped to `docs/specs/security-local.md`, or a dedicated `hosted` subagent scoped to `docs/specs/security-hosted.md`, or any of those scopes is merged back into a context that also carries another domain (rationale). -- **FAIL IF** `S8` `remote`, `local`, or `hosted` does not run on Opus and the mechanical domains on Sonnet, the stronger model on the code-reading domains, in **both** `.github/workflows/security-audit.yaml`'s `claude_args` — its `--model` sets the floor and its `--agents` raises those three domains — and `scripts/security-audit-local.sh` (rationale). Pinned by `scripts/security-audit.test.mjs`. +- **FAIL IF** `S8` `remote`, `local`, or `hosted` does not run on Opus and the mechanical domains on Sonnet, the stronger model on the code-reading domains, in **both** `.github/workflows/security-audit.yaml`'s `claude_args` — its `--model` sets the floor and its `--agents` raises those three domains — and `scripts/security-audit-local.sh`'s default Claude engine (rationale). This split is the release gate; the Codex engine never is ([Red team](#red-team)). Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S9` `.github/audit/` is missing a prompt file the workflow names, or `scripts/security-audit-local.sh` stops running the audit from those same files (rationale), or exits zero after a failed process, a missing or unfinished fragment, or any verdict other than exact `VERDICT: PASS`. Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S10` the union of the subagents' qualitative scopes does not cover every top-level path in the repository: the catch-all's remainder stops subtracting every other domain's prompt, or a second domain claims the remainder (rationale). Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S11` `.github/audit/` or `.vscode/` is outside **any** consumer of `.github/workflows/workflow-audit.yaml`'s diff window — the commit list, `own_changes`, `is_clean_merge`, and both content classifiers' refusals, whose half is *derived* from the single `WINDOW` array. The security specs are deliberately *not* watched there (rationale). @@ -121,6 +121,8 @@ Source of truth: `.github/audit/_preamble.md`; `dedupFindings` in `scripts/secur | Sink | Carries | |---|---| +| Red-team comment on the status issue, one per `codex-audit-report.yaml` run | what [Red team](#red-team) lists | +| `codex-recall` artifact | that run's canary counts, model, commit, and the two run links | | Public status issue, labeled `security-audit-status` | the run's time, link, commit, and outcome; whether the domains were skipped and when they last ran; each domain's computed verdict and its failed-check, missing-rule, malformed-line, BLOCKER, and WARNING counts; each failed check's spec and heading; how many verdict lines disagreed; whether the private filing failed; each open ledger finding's severity, kind, and dates, keyed as [Disclosure](#disclosure) allows; the latest disclosures | | Public disclosure issue, labeled `security-audit-finding`, one per ledger issue | what [Disclosure](#disclosure) lists | | Private issue in `diffplug/dormouse-embargo`, one per non-PASS run | the private report ([Outcomes and reporting](#outcomes-and-reporting)), titled with date and commit | @@ -134,7 +136,7 @@ Source of truth: `.github/audit/_preamble.md`; `dedupFindings` in `scripts/secur `EMBARGO_TOKEN` is a fine-grained PAT with Issues: write on `diffplug/dormouse-embargo` alone, minted on an admin's account. -- **FAIL IF** `S25` a public issue can carry finding text: a status or disclosure title, body, or comment is posted from anything but `scripts/security-audit-public-body.mjs` and the step's fixed failure sentence, or that builder emits anything but fixed text, counts, dates, validated run metadata, fragment names, ledger keys and severities, headings it found in the checked-out spec, files it found in the checked-out tree, PR numbers the API reports merged, and a maintainer's public summary. A failed check naming no rule its domain owes, in the line form `.github/audit/_preamble.md` fixes, is counted, never quoted (rationale). +- **FAIL IF** `S25` a public issue can carry finding text: a status or disclosure title, body, or comment is posted from anything but `scripts/security-audit-public-body.mjs` and the step's fixed failure sentence, or that builder emits anything but fixed text, counts, dates, validated run metadata and model names, fragment names, ledger keys and severities, headings it found in the checked-out spec, files it found in the checked-out tree, PR numbers the API reports merged, and a maintainer's public summary. A failed check naming no rule its domain owes, in the line form `.github/audit/_preamble.md` fixes, is counted, never quoted (rationale). - **FAIL IF** `S26` `secrets.EMBARGO_TOKEN` is referenced anywhere but the `env:` of `List open findings`, `File embargoed findings`, and `Read the findings ledger`, a workflow or job `env:` or a `$GITHUB_ENV` write could carry it to another step, any of them runs anything but `gh` and shell builtins, `List open findings` writes a title anywhere but `audit-open-findings.txt`, or `Read the findings ledger` writes anywhere but `audit-ledger-snapshot.jsonl` or keeps a title past its key and severity, a body past its opening metadata line, or a comment past a maintainer's `Fixed in` PR number or public summary (rationale). Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S27` a non-PASS run whose private filing failed or was skipped can succeed, or its status issue omits that the filing failed; or a PASS run files anything in the private tracker but ledger issues, or `scripts/security-audit-local.sh` files anything. - **FAIL IF** `S28` the `audit-transcript` artifact can upload anything but age ciphertext: its `path:` names a non-`.age` file, it uploads when `Encrypt the audit transcript` did not succeed, that step reads its recipient from anywhere but `.github/audit/transcript-recipient.txt`, or it stops running whatever the audit step's outcome (`if: always()`). @@ -206,23 +208,36 @@ Source of truth: `Verify AUDIT_PAT is provisioned` in `.github/workflows/securit - Which seeds ran, what caught them, and findings on code no seed touched stay in the `canary-transcript` artifact; a canary files them nowhere. - A canary run is a completed run on `main`, so the next scheduled run audits in full ([Skipping an unchanged audit](#skipping-an-unchanged-audit)). -- **FAIL IF** `S37` the `canary` job can push or file: it holds any write permission, runs its agent on a token but its own read-only one, keeps the checkout's credentials, runs `gh` or `git push`, or names `AUDIT_PAT` or `EMBARGO_TOKEN`; or `scripts/security-audit-local.sh canary`, which seeds a copy of the committed tree, files anything. Pinned by `scripts/security-audit-canary.test.mjs`. +- **FAIL IF** `S37` the `canary` job can push or file: it holds any write permission, runs its agent on a token but its own read-only one, keeps the checkout's credentials, runs `gh` or `git push`, or names `AUDIT_PAT` or `EMBARGO_TOKEN`; or `scripts/security-audit-local.sh canary`, which seeds a copy of the committed tree, files anything, either engine's included. Pinned by `scripts/security-audit-canary.test.mjs`. - **FAIL IF** `S38` a canary run, dispatched or weekly, can gate a release or feed the skip: the `audit` job runs in it, it runs on a ref but `main`, it uploads an `audit-state` artifact, or the gate in `.github/workflows/release.yml` can select a run with the canary's title. Pinned by `scripts/security-audit-canary.test.mjs`. - **FAIL IF** `S39` a canary publishes seed detail: the step summary or the `canary-recall` artifact carries anything but the counts and a validated run link, or the scorecard, the seeds applied, or a fragment leaves the runner but as age ciphertext to `.github/audit/transcript-recipient.txt`. Pinned by `scripts/security-audit-canary.test.mjs`. - **FAIL IF** `S40` a seed stops applying to the tree, alone or over the rest of the pool, or names an ID no rule carries, a rule no longer holding its phrase, or one its `Domain:` does not own, or a code domain has fewer than three seeds. Pinned by `scripts/security-audit-canary.test.mjs`, which `pnpm test` runs. +`scripts/security-audit-local.sh canary --key ` seeds what that CI canary run seeded, when run at its commit. + Source of truth: `seed` and `score` in `scripts/security-audit-canary.mjs`; the `canary` job in `.github/workflows/security-audit.yaml`; `run_canary` in `scripts/security-audit-local.sh`. ### Published recall -dormouse.sh/security shows a rolling recall under "How the guarantees are checked": caught over seeded across the `canary-recall` artifacts of canary runs on `main` in the 30 days before its as-of date, each run linked. It is a committed snapshot, `website/src/data/canary-recall.json`, regenerated at each release (`docs/specs/deploy.md` -> "Release checklist") (rationale). +dormouse.sh/security shows a rolling recall under "How the guarantees are checked": caught over seeded across the `canary-recall` artifacts of canary runs on `main` in the 30 days before its as-of date, each run linked. Its weekly graph draws three lines: Claude over every run; Codex over the runs a [Red team](#red-team) record pairs with; and caught by either, where a run no record pairs adds Claude's catches alone. It is a committed snapshot, `website/src/data/canary-recall.json`, regenerated at each release (`docs/specs/deploy.md` -> "Release checklist") (rationale). - **A record out of shape fails the snapshot**, never repaired or skipped: a field but the scorer's, a count not a non-negative integer, more caught than seeded, or a link but to its own run on `diffplug/dormouse`. - **Every run in the window counts as its record has it** (rationale): an unfinished domain's seeds stay seeded, caught only by what it reported, and the run shows its unfinished count. A run with no `canary-recall` artifact is listed without counts. -- The generator downloads `canary-recall` alone, never `canary-transcript`. +- The generator downloads `canary-recall` and `codex-recall` alone, never `canary-transcript`. +- **A red-team record counts only from an admin's dispatch on `main`**, paired with its canary run by link, the newest per run; one whose seeded count differs from that run's, or whose either count falls below Claude's or above the two engines' sum, fails the snapshot. Source of truth: `summarize` in `website/scripts/generate-canary-recall.js`; `CanaryRecall` in `website/src/components/CanaryRecall.tsx`. +## Red team + +**Codex audits the audit from outside its model family, and is advisory: no Codex verdict gates a release, feeds the skip, or files** (rationale). `scripts/security-audit-local.sh --engine codex` runs the same prompts on `gpt-6-sol` under the `daybreak_blue` Cyber access program, sandboxed to the working tree. **A maintainer runs it quarterly, and before any release that changes remote control or Hosted**: every domain, and a canary keyed by the latest CI canary run on `main` and scored against that run's decrypted scorecard. A Codex BLOCKER or WARNING reaches the ledger only as a maintainer's hand-filed `manual-` key ([Findings ledger](#findings-ledger)). + +`.github/workflows/codex-audit-report.yaml` publishes the run from `workflow_dispatch` inputs that carry nothing a finding could be read from: the commit, the model, each domain's computed verdict, and the canary's run ID and seeded, Codex-caught, either-caught, and unfinished counts. `scripts/security-audit-public-body.mjs` checks each against its pattern and the commit against `main`, checks the counts against that canary run's own record and commit, comments once on the status issue, and writes the `codex-recall` record [Published recall](#published-recall) pairs. + +- **FAIL IF** `S275` `.github/workflows/codex-audit-report.yaml` can gate, file, or publish past its inputs: it holds any write permission but its one job's `issues: write`, reads a secret, interpolates an input into a script, runs on a ref but `main`, posts for anyone but an admin or anything but `codexComment`'s text, or the gate in `.github/workflows/release.yml` can select its run; or `scripts/security-audit-local.sh` runs Codex unsandboxed or on another model or program. Pinned by `scripts/security-audit.test.mjs`. + +Source of truth: `run_domain` in `scripts/security-audit-local.sh`; `publishCodex` in `scripts/security-audit-public-body.mjs`; `.github/workflows/codex-audit-report.yaml`. + ## Future **Scope: audit-credential-separation** — [Credential separation](#credential-separation). diff --git a/docs/specs/security-audit.rationale.md b/docs/specs/security-audit.rationale.md index b0570c694..578244b2c 100644 --- a/docs/specs/security-audit.rationale.md +++ b/docs/specs/security-audit.rationale.md @@ -201,3 +201,13 @@ gh secret set AUDIT_PAT --env security-audit --repo diffplug/dormouse --body 'gi **Why 30 days (2026-10).** The figure should measure the audit as it runs now: its prompts, domains, and seed draw change often, and runs from months back measured a different audit. The `canary-recall` artifact's 90-day retention bounds any window, since no older run has a record left to read. **Why an unfinished domain still counts.** Dropping the run, or its unfinished domain's seeds, would raise the figure by the runs where the audit did worst; a domain that ran out of time is a miss the published recall should carry. + +## Red team + +**Why a second model family (2026-10).** Every domain, the canary's included, runs on one vendor's models, so a blind spot they share is invisible to the recall figure. Codex reads the same prompts and seeds, which isolates the model as the variable: the either line is what the two together would catch. + +**Why advisory.** A gate needs a run that is reproducible on the release tag, with credentials CI holds; the red team runs on a maintainer's machine and login, quarterly. Its value is a second opinion and a recall comparison, not a pass. + +**Why typed inputs rather than an uploaded report.** Anything the Codex run wrote could carry finding detail past the embargo; verdict words, a run ID, and four counts cannot, and a pattern check proves it. The counts are checked against the Claude canary's own record so a mistyped either count fails rather than publishing. + +**Why quarterly, and before remote control or Hosted releases.** Those are the two surfaces an attacker reaches without local access, and where the code-reading domains run longest; a release changing them is when a miss costs most. diff --git a/docs/specs/security-ci.md b/docs/specs/security-ci.md index 005976690..a309cc7b1 100644 --- a/docs/specs/security-ci.md +++ b/docs/specs/security-ci.md @@ -20,7 +20,7 @@ Renovate updates the commit pins; the generated `tend-*.yaml` pin `max-sixty/ten - **FAIL IF** `S42` `.github/workflows/release.yml` runs on anything but a pushed `v*` tag. Pinned by `scripts/workflow-lint.mjs`. - **FAIL IF** `S43` the `security-audit` job in `.github/workflows/release.yml` uses its `actions: write` for anything but dispatching `security-audit.yaml` on the release tag and watching that run. Pinned by `scripts/workflow-lint.mjs`. - **FAIL IF** `S44` `pull_request_target` appears as an `on:` trigger in any `.github/workflows/**` file other than `tend-*.yaml`. A mention in a comment is not a violation. Pinned by `scripts/workflow-lint.mjs`. -- **FAIL IF** `S45` a non-agent-managed workflow has **effective** write permissions other than the release provenance permissions `id-token: write` and `attestations: write`, or the `actions: write` granted to the `security-audit` job in `release.yml` (see "Release audit dispatch"). Effective as in the agent-managed bullet below. Pinned by `scripts/workflow-lint.mjs`. +- **FAIL IF** `S45` a non-agent-managed workflow has **effective** write permissions other than the release provenance permissions `id-token: write` and `attestations: write`, the `actions: write` granted to the `security-audit` job in `release.yml` (see "Release audit dispatch"), or the `issues: write` granted to the `publish` job in `codex-audit-report.yaml` (`docs/specs/security-audit.md` -> "Red team"). Effective as in the agent-managed bullet below. Pinned by `scripts/workflow-lint.mjs`. `.github/workflows/ci.yml` also runs actionlint and zizmor over every workflow; `.github/zizmor.yml` names each exception it accepts. diff --git a/docs/specs/security.md b/docs/specs/security.md index 921b965b1..67286a74a 100644 --- a/docs/specs/security.md +++ b/docs/specs/security.md @@ -38,6 +38,8 @@ Every section the [nightly audit](#how-the-guarantees-are-checked) checks opens **Every week, a canary audits the audit.** It seeds already-fixed or synthetic vulnerabilities into a throwaway checkout, some beside comments that justify them, removes the tests and lints that would catch them, runs the code domains over it, and publishes only how many they caught ([Canary recall](./security-audit.md#canary-recall)). +**Every quarter, and before releases that change remote control or Hosted, Codex red-teams the audit**: the same prompts and seeds on another vendor's model, published beside the nightly but never gating a release ([Red team](./security-audit.md#red-team)). + Source of truth: root scripts in `package.json`; native jobs in `.github/workflows/ci.yml`; `.github/workflows/security-audit.yaml` and its release gate in `.github/workflows/release.yml`. ## What is not defended diff --git a/scripts/security-audit-canary.mjs b/scripts/security-audit-canary.mjs index e56f5fd3d..964dd22ba 100644 --- a/scripts/security-audit-canary.mjs +++ b/scripts/security-audit-canary.mjs @@ -12,11 +12,12 @@ * says which changes are seeds. * Writes the seeds applied to /seeds.json, outside the checkout. * node scripts/security-audit-canary.mjs score --stash --scorecard --public - * [--run-url ] [--summary ] + * [--run-url ] [--summary ] [--union ] * Score the code domains' fragments in the working directory against * /seeds.json: the scorecard, with seed detail, for the encrypted * archive; the public record, counts alone; and one summary line appended - * to . + * to . `--union` names another engine's scorecard over the same + * seeds, and adds to this one's how many seeds either caught. * node scripts/security-audit-canary.mjs domains * The domains a canary runs, space-separated. * node scripts/security-audit-canary.mjs redact ... @@ -210,6 +211,18 @@ export function score(seeds, fragments) { }; } +/** + * How many seeds `scorecard` or `other` caught, both scored over the same + * seeds: a canary keyed alike over the same commit. Throws when the seeds + * differ, since a union over two different seedings counts nothing real. + */ +export function union(scorecard, other) { + const ids = (card) => card.seeds.map((s) => s.id).sort().join(); + if (ids(scorecard) !== ids(other)) throw new Error('the two scorecards seeded different seeds: key both canaries alike, over the same commit'); + const caught = new Set(other.seeds.filter((s) => s.caught).map((s) => s.id)); + return scorecard.seeds.filter((s) => s.caught || caught.has(s.id)).length; +} + /** What the canary publishes: counts and the run link, nothing a seed's detail could be read from. */ export function publicRecord(scorecard, runUrl = '') { if (runUrl && !RUN_URL_RE.test(runUrl)) throw new Error(`not a run URL: ${JSON.stringify(runUrl)}`); @@ -266,6 +279,7 @@ function main(argv) { if (command === 'score') { const seeds = JSON.parse(readFileSync(join(args.stash, 'seeds.json'), 'utf8')); const scorecard = score(seeds, readFragments()); + if (args.union) scorecard.either = union(scorecard, JSON.parse(readFileSync(args.union, 'utf8'))); const record = publicRecord(scorecard, args['run-url'] ?? ''); for (const [file, value] of [[args.scorecard, scorecard], [args.public, record]]) { mkdirSync(dirname(file), { recursive: true }); @@ -273,6 +287,7 @@ function main(argv) { } if (args.summary) appendFileSync(args.summary, `${summaryLine(record)}\n`); console.log(summaryLine(record)); + if ('either' in scorecard) console.log(`either: ${scorecard.either}/${scorecard.seeded} seeds caught by this engine or the other`); return; } if (command === 'redact') { diff --git a/scripts/security-audit-canary.test.mjs b/scripts/security-audit-canary.test.mjs index b231193ca..cc2aea4bb 100644 --- a/scripts/security-audit-canary.test.mjs +++ b/scripts/security-audit-canary.test.mjs @@ -3,7 +3,7 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { test } from 'node:test'; -import { CANARY_DOMAINS, POOL_DIR, parseSeed, publicRecord, readPool, redact, score, shuffle, summaryLine } from './security-audit-canary.mjs'; +import { CANARY_DOMAINS, POOL_DIR, parseSeed, publicRecord, readPool, redact, score, shuffle, summaryLine, union } from './security-audit-canary.mjs'; import { SENTINEL, domains, ruleIndex } from './security-audit-report.mjs'; import { repoRoot as repo, tempDir, workflowRunBlock } from './lint-kit.mjs'; @@ -435,3 +435,45 @@ test('the local canary seeds a copy, runs the code domains, scores, and files no assert.equal(execFileSync(realGit, ['status', '--porcelain'], { cwd: dir, encoding: 'utf8' }), ''); assert.ok(existsSync(join(dir, POOL_DIR, 'room.patch'))); }); + +// The red team: the same seeds under a CI run's key, scored against that +// run's scorecard, and the publishing command printed, never run. +test('the local Codex canary takes a CI run\'s key, counts either engine\'s catches, and files nothing', (t) => { + const { base, dir, bin, env } = checkout(t); + stub(bin, 'pnpm', ''); + stub(bin, 'claude', ` + const fs = require('node:fs'); + const output = process.argv[3].match(/\\*\\*Output file:\\*\\* \\x60([^\\x60]+)\\x60/)[1]; + fs.writeFileSync(output, ${JSON.stringify(fragment(warning('WARNING', 'src/room.ts:1')))}); + `); + const first = spawnSync('bash', ['scripts/security-audit-local.sh', 'canary', '3', '--key', '42'], { cwd: dir, env, encoding: 'utf8' }); + assert.equal(first.status, 0, first.stderr); + const claudeCard = first.stdout.match(/^==> scorecard: (\S+);/m)[1]; + // Codex reports nothing, so either engine caught what Claude did. + stub(bin, 'codex', ` + const fs = require('node:fs'); + const args = process.argv.slice(2); + fs.appendFileSync(process.env.CALLS, 'codex ' + args.slice(0, -1).join(' ') + '\\n'); + const output = args[args.length - 1].match(/\\*\\*Output file:\\*\\* \\x60([^\\x60]+)\\x60/)[1]; + fs.writeFileSync(output, ${JSON.stringify(fragment([]))}); + `); + stub(bin, 'claude', 'process.exit(5);'); + const result = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'codex', 'canary', '3', '--key', '42', '--claude-scorecard', claudeCard], + { cwd: dir, env, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /^canary: 0\/2 seeds caught$/m); + assert.match(result.stdout, /^either: 1\/2 seeds caught by this engine or the other$/m); + const head = execFileSync(realGit, ['rev-parse', 'HEAD'], { cwd: dir, encoding: 'utf8' }).trim(); + assert.ok(result.stdout.includes(`gh workflow run codex-audit-report.yaml -R fixture/repo --ref main -f commit=${head} -f model=gpt-6-sol -f canary_run=42 -f canary_seeded=2 -f canary_caught=0 -f canary_either=1 -f canary_unfinished=0\n`), result.stdout); + const codexCalls = calls(base).filter((c) => c.startsWith('codex ')); + assert.equal(codexCalls.length, CANARY_DOMAINS.length); + for (const c of codexCalls) assert.equal(c, 'codex exec -m gpt-6-sol --cyber-access-program daybreak_blue --sandbox workspace-write -c sandbox_workspace_write.network_access=true --ephemeral'); + assert.ok(!calls(base).some((c) => c.startsWith('gh ')), calls(base).join('\n')); +}); + +test('a union counts each seed once, and only over the same seeds', () => { + const card = (caught) => ({ seeds: Object.entries(caught).map(([id, c]) => ({ id, caught: c })) }); + assert.equal(union(card({ a: true, b: false, c: false }), card({ a: true, b: true, c: false })), 2); + assert.throws(() => union(card({ a: true, b: false }), card({ a: true, c: true })), /seeded different seeds/); +}); + diff --git a/scripts/security-audit-local.sh b/scripts/security-audit-local.sh index 75fbd252a..529b2afb5 100755 --- a/scripts/security-audit-local.sh +++ b/scripts/security-audit-local.sh @@ -14,12 +14,24 @@ # scripts/security-audit-local.sh # the deterministic checks, then every domain # scripts/security-audit-local.sh remote # one domain # scripts/security-audit-local.sh github-state # the deterministic checks alone -# scripts/security-audit-local.sh canary [count] # recall on seeded vulnerabilities +# scripts/security-audit-local.sh canary [count] [--key ] [--claude-scorecard ] +# # recall on seeded vulnerabilities +# scripts/security-audit-local.sh --engine codex [any of the above] # # `canary` measures rather than audits (docs/specs/security-audit.md -> "Canary # recall"): it copies the committed tree to a temporary directory, seeds it from # `.github/audit/canaries/` as CI does, runs the code domains there, and # scores their fragments. Your checkout is never touched, and nothing is filed. +# `--key` takes a CI canary run's seed key, its run ID, so the same commit seeds +# the same seeds that run did. +# +# `--engine codex` runs the same prompts on Codex instead of Claude: the red +# team (docs/specs/security-audit.md -> "Red team"), advisory, never the +# release gate. Its canary takes `--claude-scorecard`, the decrypted +# `scorecard.json` of the CI canary run its `--key` names, to count the seeds +# either engine caught. It ends by printing the `gh workflow run +# codex-audit-report.yaml` command that publishes its verdicts and counts; +# running that is yours to do, since this script files nothing. # # The deterministic GitHub-state check (`scripts/github-state-check.mjs`) runs # first whenever a domain that reads its fragment does — `ci-and-secrets` and @@ -41,8 +53,23 @@ export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-diffplug/dormouse}" # next run an expired deadline. export RUNNER_TEMP="${RUNNER_TEMP:-$(mktemp -d)}" -if ! command -v claude >/dev/null 2>&1; then - echo "error: the \`claude\` CLI is not on PATH." >&2 +ENGINE=claude +if [ "${1:-}" = --engine ]; then + [ $# -ge 2 ] || { echo "error: --engine needs claude or codex" >&2; exit 64; } + ENGINE="$2" + shift 2 +fi +case "$ENGINE" in + claude|codex) ;; + *) echo "error: unknown engine '$ENGINE' (claude|codex)" >&2; exit 64 ;; +esac +# The red team's model and Cyber access program. Pinned here alone: the +# Codex engine has no CI counterpart to agree with. +CODEX_MODEL=gpt-6-sol +CODEX_PROGRAM=daybreak_blue + +if ! command -v "$ENGINE" >/dev/null 2>&1; then + echo "error: the \`$ENGINE\` CLI is not on PATH." >&2 exit 1 fi @@ -82,15 +109,31 @@ run_domain() { local model_args="--model sonnet" case "$domain" in remote|local|hosted) model_args="--model opus" ;; esac - echo "==> $domain -> $out${model_args:+ ($model_args)}" - rm -f "$out" - # shellcheck disable=SC2086 - if ! claude -p "$(cat "$AUDIT_DIR/_preamble.md"; echo; cat "$AUDIT_DIR/$domain.md")" \ - $model_args \ - --allowed-tools "Read,Write,Edit,Bash,Grep,Glob" \ - --disallowed-tools "Task,Agent,Workflow"; then - echo "==> $domain auditor process failed" >&2 - return 1 + if [ "$ENGINE" = codex ]; then + # Sandboxed to the working tree, with the network `ci-and-secrets` needs + # for live GitHub state; a check the sandbox keeps from the operator's `gh` + # login comes back UNVERIFIABLE, never PASS. The leading line stands in + # for the Claude run's denied `Task`: the domain runs every work stream + # itself. + echo "==> $domain -> $out (codex: $CODEX_MODEL, $CODEX_PROGRAM)" + rm -f "$out" + if ! codex exec -m "$CODEX_MODEL" --cyber-access-program "$CODEX_PROGRAM" \ + --sandbox workspace-write -c sandbox_workspace_write.network_access=true --ephemeral \ + "$(echo "Do not delegate: run every work stream yourself."; echo; cat "$AUDIT_DIR/_preamble.md"; echo; cat "$AUDIT_DIR/$domain.md")" $domain auditor process failed" >&2 + return 1 + fi + else + echo "==> $domain -> $out${model_args:+ ($model_args)}" + rm -f "$out" + # shellcheck disable=SC2086 + if ! claude -p "$(cat "$AUDIT_DIR/_preamble.md"; echo; cat "$AUDIT_DIR/$domain.md")" \ + $model_args \ + --allowed-tools "Read,Write,Edit,Bash,Grep,Glob" \ + --disallowed-tools "Task,Agent,Workflow"; then + echo "==> $domain auditor process failed" >&2 + return 1 + fi fi if [ ! -s "$out" ]; then echo "==> $domain produced no fragment — in CI that is an INCONCLUSIVE audit, not a FAIL" >&2 @@ -100,7 +143,34 @@ run_domain() { # The verdict CI reports, computed from the fragment's lines: zero only on # PASS, which also needs its sentinel and a first line of exactly # `VERDICT: PASS`. It prints why anything else is not. - node scripts/security-audit-report.mjs check "$out" >&2 + local report status=0 + report=$(node scripts/security-audit-report.mjs check "$out") || status=$? + printf '%s\n' "$report" >&2 + VERDICTS="$VERDICTS $domain=$(printf '%s\n' "$report" | sed -n '1s/^[^:]*: \([A-Z]*\).*/\1/p')" + return "$status" +} + +# Each domain's computed verdict, `domain=VERDICT`, for the red team's report; +# a domain that left no fragment is INCONCLUSIVE, as CI computes it. +VERDICTS="" +run_domain_recorded() { + local domain="$1" before="$VERDICTS" status=0 + run_domain "$domain" || status=$? + [ "$VERDICTS" != "$before" ] || VERDICTS="$VERDICTS $domain=INCONCLUSIVE" + return "$status" +} + +# The command that publishes the red team's report, printed, never run. +print_codex_report() { + local commit + commit=$(git rev-parse HEAD 2>/dev/null) || commit="" + echo + echo "==> advisory: a Codex verdict never gates a release (docs/specs/security-audit.md -> \"Red team\")." + if [ -n "$(git status --porcelain 2>/dev/null)" ]; then + echo "==> the checkout has uncommitted changes; these results are of that tree, not of $commit." >&2 + fi + echo "==> to publish them:" + echo "gh workflow run codex-audit-report.yaml -R $GITHUB_REPOSITORY --ref main -f commit=$commit -f model=$CODEX_MODEL $*" } # The open ledger findings every domain re-verifies, as CI's `List open @@ -136,23 +206,52 @@ run_github_state() { } # The canary, in a copy of the committed tree: seeded, audited by the code -# domains, then scored. `CANARY_KEY` picks the seeds; -# the same key and pool seed the same way. +# domains, then scored. `--key` (or `CANARY_KEY`) picks the seeds; the same +# key and pool seed the same way, so a CI canary run's ID over its commit +# seeds what it seeded (`seed --key "$GITHUB_RUN_ID"` in the workflow). run_canary() { - local count="${1:-8}" tree stash domain + local count=8 key="${CANARY_KEY:-}" claude_card="" tree stash domain + while [ $# -gt 0 ]; do + case "$1" in + --key) key="${2:-}"; shift 2 || { echo "error: --key needs a value" >&2; return 64; } ;; + --claude-scorecard) claude_card="${2:-}"; shift 2 || { echo "error: --claude-scorecard needs a file" >&2; return 64; } ;; + *[!0-9]*|'') echo "error: unexpected canary argument '$1'" >&2; return 64 ;; + *) count="$1"; shift ;; + esac + done + if [ -n "$claude_card" ]; then + [ "$ENGINE" = codex ] || { echo "error: --claude-scorecard is the Codex engine's" >&2; return 64; } + [ -n "$key" ] || { echo "error: --claude-scorecard needs the --key of the run it scored" >&2; return 64; } + [ -f "$claude_card" ] || { echo "error: no scorecard at $claude_card" >&2; return 1; } + claude_card=$(cd "$(dirname "$claude_card")" && pwd)/$(basename "$claude_card") + fi + key="${key:-$(date +%s)}" tree=$(mktemp -d) stash=$(mktemp -d) git archive HEAD | tar -x -C "$tree" echo "==> canary: seeding $count in $tree" - (cd "$tree" && node scripts/security-audit-canary.mjs seed --key "${CANARY_KEY:-$(date +%s)}" --count "$count" --stash "$stash" && + (cd "$tree" && node scripts/security-audit-canary.mjs seed --key "$key" --count "$count" --stash "$stash" && pnpm install --frozen-lockfile >/dev/null) # Run here rather than through the copy's runner, which would read the real # findings ledger: a canary hands its domains an empty list, as CI does. (cd "$tree" && : > audit-open-findings.txt && for domain in $(node scripts/security-audit-canary.mjs domains); do run_domain "$domain" || true; done) (cd "$tree" && node scripts/security-audit-canary.mjs score --stash "$stash" \ - --scorecard "$stash/scorecard.json" --public "$stash/canary-recall.json") + --scorecard "$stash/scorecard.json" --public "$stash/canary-recall.json" ${claude_card:+--union "$claude_card"}) echo "==> scorecard: $stash/scorecard.json; seeded tree: $tree" + [ "$ENGINE" = codex ] || return 0 + case "$key" in + *[!0-9]*) echo "==> keyed by no CI run, so there is no canary run to publish against." ; return 0 ;; + esac + if [ -z "$claude_card" ]; then + echo "==> to publish, re-run with --claude-scorecard: the scorecard.json in run $key's decrypted canary-transcript." + return 0 + fi + # shellcheck disable=SC2046,SC2016 + print_codex_report $(node -e ' + const card = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); + console.log(`-f canary_run=${process.argv[2]} -f canary_seeded=${card.seeded} -f canary_caught=${card.caught} -f canary_either=${card.either} -f canary_unfinished=${card.unfinished.length}`); + ' "$stash/scorecard.json" "$key") } if [ $# -gt 0 ]; then @@ -163,7 +262,8 @@ if [ $# -gt 0 ]; then ci-and-secrets|supply-chain) run_github_state || status=1 ;; esac write_open_findings - run_domain "$1" || status=$? + run_domain_recorded "$1" || status=$? + [ "$ENGINE" = claude ] || print_codex_report "-f verdicts='${VERDICTS# }'" exit "$status" fi @@ -174,11 +274,12 @@ status=0 run_github_state || status=1 write_open_findings for domain in supply-chain ci-and-secrets remote local hosted; do - run_domain "$domain" || status=1 + run_domain_recorded "$domain" || status=1 done echo echo "==> fragments:" ls -la audit-*.md 2>/dev/null || echo " (none)" echo "==> each verdict above is the one CI computes; there is no merge here, so read the fragments directly." +[ "$ENGINE" = claude ] || print_codex_report "-f verdicts='${VERDICTS# }'" exit "$status" diff --git a/scripts/security-audit-public-body.mjs b/scripts/security-audit-public-body.mjs index b87039bdc..799ca81fb 100644 --- a/scripts/security-audit-public-body.mjs +++ b/scripts/security-audit-public-body.mjs @@ -23,6 +23,17 @@ * $GITHUB_REPOSITORY. Once everything is published it writes `passed=true` to * only when the run passed, its findings were filed, and the ledger was * read; `passed=false` otherwise. + * + * Usage: node scripts/security-audit-public-body.mjs --engine codex + * --date --run-url --commit --model + * --verdicts --canary-run --canary-seeded + * --canary-caught --canary-either --canary-unfinished + * --record + * The red team's report (docs/specs/security-audit.md -> "Red team"), from + * `.github/workflows/codex-audit-report.yaml`'s inputs: one comment on the + * status issue, and, for a canary, its recall record written to . Every + * input is checked against its pattern, the counts against the CI canary run + * they share seeds with, and nothing else is published. */ import { spawnSync } from 'node:child_process'; @@ -31,7 +42,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { clampIssueBody } from './clamp-issue-body.mjs'; -import { dedupFindings, judgeFragment, readFileStatus, readFragments, ruleIndex } from './security-audit-report.mjs'; +import { dedupFindings, domains, judgeFragment, readFileStatus, readFragments, ruleIndex } from './security-audit-report.mjs'; /** The label that finds the one status issue, and the one every disclosure carries. */ export const STATUS_LABEL = 'security-audit-status'; @@ -334,6 +345,91 @@ export function statusBody({ status, fileStatus, date, runUrl, commit, repo, fra /** The outcome an earlier run recorded in the status issue's first line, or null. */ export const previousOutcome = (body) => (typeof body === 'string' ? body.match(STATUS_MARKER)?.[1] ?? null : null); +// --- The red team ------------------------------------------------------------- + +/** The marker opening every red-team comment on the status issue. */ +export const CODEX_MARKER = ''; +const MODEL = /^[a-z0-9][a-z0-9.-]{0,63}$/; +const COUNT = /^(?:0|[1-9]\d{0,5})$/; + +/** `domain=VERDICT …` read into pairs, every domain one the prompts name, each at most once. */ +export function parseVerdicts(text, root = '.') { + const known = domains(root).map((d) => d.domain); + const out = []; + for (const word of text.split(/\s+/).filter(Boolean)) { + const [, domain, verdict] = valid(word, /^([a-z-]+)=(PASS|FAIL|INCONCLUSIVE)$/, 'verdict').match(/^([a-z-]+)=(\w+)$/); + if (!known.includes(domain)) throw new Error(`invalid verdict: no domain ${JSON.stringify(domain)}`); + if (out.some((v) => v.domain === domain)) throw new Error(`invalid verdict: ${domain} twice`); + out.push({ domain, verdict }); + } + return out; +} + +/** + * The red team's canary counts, or null when none were given; throws unless + * all are, each a count, caught no more than either and either no more than + * seeded, and no more unfinished domains than the `domains` a canary runs. + */ +export function parseCanary({ run = '', seeded = '', caught = '', either = '', unfinished = '' }, domains) { + const given = [run, seeded, caught, either, unfinished]; + if (given.every((v) => v === '')) return null; + valid(run, /^[1-9]\d{0,19}$/, 'canary run'); + const [s, c, e, u] = [seeded, caught, either, unfinished].map((v, i) => Number(valid(v, COUNT, ['canary seeded', 'canary caught', 'canary either', 'canary unfinished'][i]))); + if (c > e || e > s) throw new Error(`invalid canary counts: caught ${c}, either ${e}, seeded ${s}`); + if (u > domains) throw new Error(`invalid canary counts: ${u} unfinished domains of ${domains}`); + return { run, seeded: s, caught: c, either: e, unfinished: u }; +} + +/** + * Checks the red team's canary against the CI canary run it shares seeds with + * — `claudeRun`, the API's view of it, and `claude`, its `canary-recall` + * record, read by `validateRecord` — and returns that record's counts: a run + * titled `canaryTitle` of `security-audit.yaml` on `main` at the same commit, + * as many seeds, and an `either` that neither engine's own count exceeds nor + * their sum falls short of. + */ +export function checkAgainstClaude(canary, { commit, repo, claudeRun, claude, canaryTitle, validateRecord }) { + const where = `canary run ${canary.run}`; + if (claudeRun?.display_title !== canaryTitle || claudeRun.head_branch !== 'main' || claudeRun.path !== '.github/workflows/security-audit.yaml' + || claudeRun.head_repository?.full_name !== repo) throw new Error(`${where} is not a canary run of security-audit.yaml on main`); + if (claudeRun.head_sha !== commit) throw new Error(`${where} seeded ${claudeRun.head_sha}, not ${commit}: the same key seeds the same seeds only over the same commit`); + const counts = validateRecord(claude, Number(canary.run)); + if (counts.seeded !== canary.seeded) throw new Error(`${where} seeded ${counts.seeded}, not ${canary.seeded}`); + if (canary.either < counts.caught || canary.either > counts.caught + canary.caught) { + throw new Error(`either ${canary.either} cannot follow from Claude's ${counts.caught} and Codex's ${canary.caught} of ${canary.seeded}`); + } + return counts; +} + +const percent = (n, of) => `${Math.round((n / of) * 100)}%`; + +/** The comment the red team's report posts on the status issue. */ +export function codexComment({ date, runUrl, commit, model, repo, verdicts, canary, claude }) { + valid(date, /^\d{4}-\d\d-\d\dT\d\d:\d\dZ$/, 'date'); + valid(repo, /^[\w.-]+\/[\w.-]+$/, 'repository'); + valid(runUrl, new RegExp(`^https://github\\.com/${repo.replace(/[.]/g, '\\.')}/actions/runs/\\d+$`), 'run URL'); + valid(commit, /^[0-9a-f]{40}$/, 'commit'); + valid(model, MODEL, 'model'); + const lines = [CODEX_MARKER, '', '## Red team: Codex (advisory)', '', + `Reported at ${date} · model \`${model}\` · audited commit [\`${commit.slice(0, 7)}\`](https://github.com/${repo}/commit/${commit}) · [report run](${runUrl})`, '', + 'A maintainer ran these prompts on Codex. Its verdicts never gate a release; the nightly Claude audit does.']; + if (verdicts.length > 0) { + lines.push('', '| Domain | Verdict |', '| --- | --- |', ...verdicts.map((v) => `| \`${v.domain}\` | ${v.verdict} |`)); + } + if (canary) { + const claudeUrl = `https://github.com/${repo}/actions/runs/${canary.run}`; + lines.push('', `**Canary**, on the seeds of [canary run ${canary.run}](${claudeUrl}): Claude caught ${claude.caught} of ${canary.seeded} (${percent(claude.caught, canary.seeded)}), Codex ${canary.caught} (${percent(canary.caught, canary.seeded)}), and the two together ${canary.either} (${percent(canary.either, canary.seeded)}).`); + if (canary.unfinished > 0) lines.push('', `${plural(canary.unfinished, 'Codex domain')} left no finished report; ${canary.unfinished === 1 ? 'its' : 'their'} seeds count as missed.`); + } + return clampIssueBody(`${lines.join('\n')}\n`); +} + +/** The red team's recall record, which website/scripts/generate-canary-recall.js reads. */ +export function codexRecord({ runUrl, commit, model, repo, canary }) { + return { claude_run_url: `https://github.com/${repo}/actions/runs/${canary.run}`, commit, model, + seeded: canary.seeded, caught: canary.caught, either: canary.either, unfinished: canary.unfinished, run_url: runUrl }; +} + // --- Publishing --------------------------------------------------------------- /** `gh` on the workflow token; throws on failure, so nothing is published half-checked. */ @@ -421,12 +517,61 @@ function publish(args) { return ok; } -function main(argv) { +/** + * The red team's report: every input checked before anything is posted. The + * canary's contract is imported here alone, so the nightly's own publishing + * never loads the recall generator. + */ +async function publishCodex(args) { + const { CANARY_DOMAINS } = await import('./security-audit-canary.mjs'); + const { CANARY_TITLE, validateRecord } = await import('../website/scripts/generate-canary-recall.js'); + const repo = valid(process.env.GITHUB_REPOSITORY, /^[\w.-]+\/[\w.-]+$/, 'repository'); + const R = ['--repo', repo]; + const commit = valid(args.commit, /^[0-9a-f]{40}$/, 'commit'); + valid(args.model, MODEL, 'model'); + const verdicts = parseVerdicts(args.verdicts ?? ''); + const canary = parseCanary({ run: args['canary-run'], seeded: args['canary-seeded'], caught: args['canary-caught'], + either: args['canary-either'], unfinished: args['canary-unfinished'] }, CANARY_DOMAINS.length); + if (verdicts.length === 0 && canary === null) throw new Error('nothing to report: give verdicts, canary counts, or both'); + // An admin's dispatch alone: the bot's `push` can dispatch a workflow too. + const actor = valid(process.env.GITHUB_TRIGGERING_ACTOR, /^[A-Za-z0-9-]+(?:\[bot\])?$/, 'triggering actor'); + const role = gh(['api', `repos/${repo}/collaborators/${actor}/permission`, '--jq', '.role_name']).trim(); + if (role !== 'admin') throw new Error(`${actor} holds ${role || 'no role'} here; only an admin publishes a red-team report`); + // Only a commit on this repository's `main`, so a report names code that shipped there. + const compare = JSON.parse(gh(['api', `repos/${repo}/compare/${commit}...main`, '--jq', '{status}'])); + if (!['identical', 'ahead'].includes(compare.status)) throw new Error(`${commit} is not on main`); + let claude = null; + if (canary) { + const claudeRun = JSON.parse(gh(['api', `repos/${repo}/actions/runs/${canary.run}`])); + const dir = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), 'canary-recall-')); + gh(['run', 'download', canary.run, ...R, '-n', 'canary-recall', '-D', dir]); + claude = checkAgainstClaude(canary, { commit, repo, claudeRun, claude: JSON.parse(readFileSync(join(dir, 'canary-recall.json'), 'utf8')), + canaryTitle: CANARY_TITLE, validateRecord }); + } + const fields = { date: args.date, runUrl: args['run-url'], commit, model: args.model, repo, verdicts, canary, claude }; + const body = codexComment(fields); + const record = canary ? codexRecord(fields) : null; + const statuses = JSON.parse(gh(['issue', 'list', ...R, '--label', STATUS_LABEL, '--state', 'open', '--limit', '100', '--json', 'number'])) + .sort((a, b) => a.number - b.number); + // The nightly audit owns the status issue: with none, there is nothing to comment on. + if (statuses.length === 0) throw new Error(`no open issue labeled ${STATUS_LABEL}`); + if (record) writeFileSync(valid(args.record, /\S/, 'record file'), `${JSON.stringify(record, null, 2)}\n`); + const dir = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), 'codex-public-')); + gh(['issue', 'comment', String(statuses[0].number), ...R, '--body-file', bodyFile(dir, 'codex.md', body)]); + console.log(`Status issue #${statuses[0].number}: red-team report posted${record ? ', recall record written' : ''}.`); +} + +async function main(argv) { const args = {}; for (let i = 2; i < argv.length; i += 2) { if (!argv[i].startsWith('--')) throw new Error(`unexpected argument: ${argv[i]}`); args[argv[i].slice(2)] = argv[i + 1] ?? ''; } + if (args.engine !== undefined) { + valid(args.engine, /^codex$/, 'engine'); + await publishCodex(args); + return; + } const ok = publish(args); // Written last: a builder that throws hands on nothing, which the step reads as a failure. writeFileSync(valid(args.outputs, /\S/, 'outputs file'), `passed=${ok}\n`); diff --git a/scripts/security-audit.test.mjs b/scripts/security-audit.test.mjs index 7a5727a2f..f3d871472 100644 --- a/scripts/security-audit.test.mjs +++ b/scripts/security-audit.test.mjs @@ -1522,6 +1522,36 @@ for (const [arg, runsState, runsDomain] of [['github-state', true, false], ['ci- }); } +// The red team: the same prompts on Codex, each domain's computed verdict +// gathered into the publishing command, which is printed and never run. +for (const [args, verdicts, expected] of [[['remote'], 'remote=PASS', 0], [[], 'supply-chain=PASS ci-and-secrets=PASS remote=PASS local=INCONCLUSIVE hosted=PASS', 1]]) { + test(`local runner, Codex engine: ${args[0] ?? 'every domain'}`, (t) => { + const { dir, env } = fixture(t); + localRunnerFixture(dir); + stubGh(dir); + stub(dir, 'claude', 'process.exit(5);'); + // `local` leaves no fragment, which computes INCONCLUSIVE. + stub(dir, 'codex', ` + const fs = require('node:fs'); + const argv = process.argv.slice(2); + const output = argv[argv.length - 1].match(/\\*\\*Output file:\\*\\* \\x60([^\\x60]+)\\x60/)[1]; + if (output !== 'audit-local.md') fs.writeFileSync(output, ${JSON.stringify(Object.fromEntries(fragments.map((f) => [f, fragmentText(f)])))}[output]); + `); + const result = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'codex', ...args], { cwd: dir, env, encoding: 'utf8' }); + assert.equal(result.status, expected, result.stderr); + assert.match(result.stdout, new RegExp(`^gh workflow run codex-audit-report\\.yaml -R fixture/repo --ref main -f commit=\\S+ -f model=gpt-6-sol -f verdicts='${verdicts}'$`, 'm')); + assert.deepEqual(ghCalls(dir).map(({ args: a }) => a.slice(0, 2).join(' ')).filter((c) => c !== 'issue list'), []); + }); +} + +test('local runner: an unknown engine is refused before anything runs', (t) => { + const { dir, env } = fixture(t); + localRunnerFixture(dir); + const result = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'gemini'], { cwd: dir, env, encoding: 'utf8' }); + assert.equal(result.status, 64); + assert.match(result.stderr, /unknown engine 'gemini'/); +}); + // The orchestrator prompt's two sanctioned shell blocks, executed as shipped. // They are the only place that decides whether a domain reported, and prose is // not a control: run 35205193090 merged a fragment its domain was still @@ -1782,6 +1812,109 @@ test('the model split holds in CI and locally, over the same prompt files', () = assert.match(local, /cat "\$AUDIT_DIR\/_preamble\.md"; echo; cat "\$AUDIT_DIR\/\$domain\.md"/); }); +// "Red team": the Codex engine is pinned and sandboxed, and its report +// workflow publishes only what its builder checked, for an admin, from main. +const codexWorkflow = readFileSync(join(repo, '.github/workflows/codex-audit-report.yaml'), 'utf8'); + +test('the red team runs sandboxed on its pinned model and program, and gates nothing', () => { + assert.match(local, /^CODEX_MODEL=gpt-6-sol$/m); + assert.match(local, /^CODEX_PROGRAM=daybreak_blue$/m); + const exec = local.match(/codex exec [^\n]*\\\n[^\n]*/)?.[0] ?? ''; + assert.match(exec, /-m "\$CODEX_MODEL" --cyber-access-program "\$CODEX_PROGRAM"/); + assert.match(exec, /--sandbox workspace-write /); + assert.doesNotMatch(local, /danger-full-access|dangerously-bypass/); + // The release gate lists runs of security-audit.yaml alone. + assert.doesNotMatch(release, /codex/i); + const doc = YAML.parse(codexWorkflow); + assert.deepEqual(Object.keys(doc.on), ['workflow_dispatch']); + assert.deepEqual(doc.permissions, {}); + assert.deepEqual(Object.keys(doc.jobs), ['publish']); + assert.deepEqual(doc.jobs.publish.permissions, { contents: 'read', issues: 'write', actions: 'read' }); + assert.equal(doc.jobs.publish.environment, undefined); + assert.doesNotMatch(codexWorkflow, /secrets\./); + const steps = doc.jobs.publish.steps; + for (const step of steps) assert.doesNotMatch(step.run ?? '', /\$\{\{/, `${step.name ?? step.uses} interpolates into its script`); + assert.equal(steps[0].name, 'Refuse any ref but main'); + for (const [ref, status] of [['refs/heads/main', 0], ['refs/tags/v1.2.3', 1], ['refs/heads/feature', 1]]) { + assert.equal(spawnSync('bash', ['-c', steps[0].run], { env: { ...process.env, GITHUB_REF: ref } }).status, status, ref); + } + const posting = steps.filter((step) => /\bgh\b|security-audit-public-body/.test(step.run ?? '')); + assert.deepEqual(posting.map((step) => step.name), ['Publish the red-team report']); + assert.match(posting[0].run, /node scripts\/security-audit-public-body\.mjs --engine codex /); + assert.equal(doc.jobs.publish.steps.find((step) => step.uses?.startsWith('actions/checkout@')).with['persist-credentials'], false); +}); + +/** The builder's red-team path in a fixture tree, over a `gh` stub answering what it reads. */ +function codexFixture(t, { role = 'admin', compare = 'ahead', run = {}, record = {} } = {}) { + const { dir, env } = fixture(t); + copyFileSync(join(repo, 'scripts/security-audit-canary.mjs'), join(dir, 'scripts/security-audit-canary.mjs')); + mkdirSync(join(dir, 'website/scripts'), { recursive: true }); + copyFileSync(join(repo, 'website/scripts/generate-canary-recall.js'), join(dir, 'website/scripts/generate-canary-recall.js')); + const claudeRun = { display_title: 'security-audit canary (recall measurement, not an audit)', head_branch: 'main', + path: '.github/workflows/security-audit.yaml', head_repository: { full_name: 'fixture/repo' }, head_sha: COMMIT, ...run }; + const canaryRecall = { seeded: 8, caught: 6, unfinished: 0, run_url: 'https://github.com/diffplug/dormouse/actions/runs/77', ...record }; + stub(dir, 'gh', ` + const fs = require('node:fs'); + const args = process.argv.slice(2); + const opt = (name) => (args.includes(name) ? args[args.indexOf(name) + 1] : null); + fs.appendFileSync('gh-calls.jsonl', JSON.stringify({ args, body: opt('--body-file') && fs.readFileSync(opt('--body-file'), 'utf8') }) + '\\n'); + const path = args[0] === 'api' ? args[1] : ''; + if (/\\/collaborators\\/[^/]+\\/permission$/.test(path)) process.stdout.write(${JSON.stringify(role)} + '\\n'); + else if (/\\/compare\\//.test(path)) process.stdout.write(JSON.stringify({ status: ${JSON.stringify(compare)} })); + else if (/\\/actions\\/runs\\/77$/.test(path)) process.stdout.write(${JSON.stringify(JSON.stringify(claudeRun))}); + else if (args[0] === 'run' && args[1] === 'download') fs.writeFileSync(opt('-D') + '/canary-recall.json', ${JSON.stringify(JSON.stringify(canaryRecall))}); + else if (args[0] === 'issue' && args[1] === 'list') process.stdout.write(JSON.stringify([{ number: 9 }, { number: 4 }])); + else if (!(args[0] === 'issue' && args[1] === 'comment')) process.exit(3); + `); + return { dir, env: { ...env, GITHUB_TRIGGERING_ACTOR: 'maintainer' } }; +} + +const CODEX_ARGS = ['--engine', 'codex', '--date', '2026-10-20T12:00Z', '--run-url', 'https://github.com/fixture/repo/actions/runs/5', + '--commit', COMMIT, '--model', 'gpt-6-sol', '--verdicts', 'remote=PASS local=FAIL', '--canary-run', '77', '--canary-seeded', '8', + '--canary-caught', '5', '--canary-either', '7', '--canary-unfinished', '0', '--record', 'codex-recall.json']; +const runCodex = (dir, env, args = CODEX_ARGS) => spawnSync(process.execPath, ['scripts/security-audit-public-body.mjs', ...args], { cwd: dir, env, encoding: 'utf8' }); +const withArg = (name, value) => CODEX_ARGS.map((a, i) => (CODEX_ARGS[i - 1] === name ? value : a)); + +test('the red-team report comments once on the status issue and writes its recall record', (t) => { + const { dir, env } = codexFixture(t); + const result = runCodex(dir, env); + assert.equal(result.status, 0, result.stderr); + const calls = readFileSync(join(dir, 'gh-calls.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l)); + const writes = calls.filter(({ args }) => args[0] === 'issue' && args[1] !== 'list'); + assert.deepEqual(writes.map(({ args }) => args.slice(0, 3)), [['issue', 'comment', '4']]); + assert.match(writes[0].body, /^\n/); + assert.match(writes[0].body, /Claude caught 6 of 8 \(75%\), Codex 5 \(63%\), and the two together 7 \(88%\)\./); + assert.match(writes[0].body, /\| `local` \| FAIL \|/); + assert.deepEqual(JSON.parse(readFileSync(join(dir, 'codex-recall.json'), 'utf8')), { + claude_run_url: 'https://github.com/fixture/repo/actions/runs/77', commit: COMMIT, model: 'gpt-6-sol', + seeded: 8, caught: 5, either: 7, unfinished: 0, run_url: 'https://github.com/fixture/repo/actions/runs/5' }); +}); + +for (const [name, options, args, message] of [ + ['a dispatch by a non-admin', { role: 'write' }, CODEX_ARGS, /only an admin/], + ['a commit off main', { compare: 'diverged' }, CODEX_ARGS, /is not on main/], + ['a canary run that is not a canary', { run: { display_title: 'security-audit' } }, CODEX_ARGS, /is not a canary run/], + ['a canary run at another commit', { run: { head_sha: 'f'.repeat(40) } }, CODEX_ARGS, /the same key seeds the same seeds only over the same commit/], + ['other seeds', { record: { seeded: 6, caught: 5 } }, CODEX_ARGS, /seeded 6, not 8/], + ['either below Claude', {}, withArg('--canary-either', '5'), /cannot follow/], + ['either above the sum', { record: { caught: 1 } }, CODEX_ARGS, /cannot follow/], + ['a finding typed as a verdict', {}, withArg('--verdicts', 'remote=FAIL:token-in-log'), /invalid verdict/], + ['an unknown domain', {}, withArg('--verdicts', 'secrets=PASS'), /no domain/], + ['a model name carrying text', {}, withArg('--model', 'gpt-6-sol see src/x.ts:12'), /invalid model/], + ['caught above either', {}, withArg('--canary-caught', '8'), /invalid canary counts/], + ['half the canary counts', {}, withArg('--canary-unfinished', ''), /invalid canary unfinished/], +]) { + test(`the red-team report posts nothing for ${name}`, (t) => { + const { dir, env } = codexFixture(t, options); + const result = runCodex(dir, env, args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, message); + const calls = existsSync(join(dir, 'gh-calls.jsonl')) ? readFileSync(join(dir, 'gh-calls.jsonl'), 'utf8') : ''; + assert.doesNotMatch(calls, /"comment"/); + assert.ok(!existsSync(join(dir, 'codex-recall.json'))); + }); +} + // "Domains": dormouse.sh/security publishes each domain with the specs its // scope claims, read from the same prompts the audit runs. test('the published audit index lists every domain with its specs', async () => { diff --git a/scripts/security-rule-ids.json b/scripts/security-rule-ids.json index f0289c602..02d04c86a 100644 --- a/scripts/security-rule-ids.json +++ b/scripts/security-rule-ids.json @@ -1,5 +1,5 @@ { - "next": 275, + "next": 276, "retired": [ 265 ] diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index a863fdb0a..5e468e59f 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -5,7 +5,7 @@ "docs/compatible-agents.md": 1800, "docs/specs/alert.md": 7100, "docs/specs/auto-update.md": 1000, - "docs/specs/deploy.md": 2200, + "docs/specs/deploy.md": 2250, "docs/specs/dor-browser.md": 5800, "docs/specs/dor-cli.md": 5150, "docs/specs/dor-tool.md": 6550, @@ -24,13 +24,13 @@ "docs/specs/remote-network.md": 2400, "docs/specs/remote-security-model.md": 5700, "docs/specs/reopen.md": 1200, - "docs/specs/security-audit.md": 4750, + "docs/specs/security-audit.md": 5150, "docs/specs/security-ci.md": 3400, "docs/specs/security-hosted.md": 3050, "docs/specs/security-local.md": 6100, "docs/specs/security-remote.md": 7200, "docs/specs/security-supply-chain.md": 1650, - "docs/specs/security.md": 2750, + "docs/specs/security.md": 2800, "docs/specs/shortcuts.md": 1150, "docs/specs/standalone.md": 6450, "docs/specs/terminal-context.md": 950, diff --git a/scripts/workflow-lint.mjs b/scripts/workflow-lint.mjs index 44dc062c3..4baceb655 100644 --- a/scripts/workflow-lint.mjs +++ b/scripts/workflow-lint.mjs @@ -130,7 +130,8 @@ export function check(inputs) { const extra = held.filter((s) => !AGENT_WRITES.includes(s)); if (extra.length) fail(TEND, `${file} job \`${jobId}\` holds write on ${extra.join(', ')}`); } else { - const allowed = file === 'release.yml' && jobId === 'security-audit' ? [...RELEASE_WRITES, 'actions'] : RELEASE_WRITES; + const allowed = file === 'release.yml' && jobId === 'security-audit' ? [...RELEASE_WRITES, 'actions'] + : file === 'codex-audit-report.yaml' && jobId === 'publish' ? [...RELEASE_WRITES, 'issues'] : RELEASE_WRITES; const extra = held.filter((s) => !allowed.includes(s)); if (extra.length) fail(POLICIES, `${file} job \`${jobId}\` holds write on ${extra.join(', ')}`); } diff --git a/scripts/workflow-lint.test.mjs b/scripts/workflow-lint.test.mjs index b6949161f..19e2bd854 100644 --- a/scripts/workflow-lint.test.mjs +++ b/scripts/workflow-lint.test.mjs @@ -36,6 +36,8 @@ const cases = [ ['a non-agent workflow with write-all', edit('ci.yml', 'permissions:\n contents: read', 'permissions: write-all'), 'holds write on *'], ['release.yml build job writes packages', edit('release.yml', ' id-token: write\n attestations: write\n strategy:', ' id-token: write\n attestations: write\n packages: write\n strategy:'), 'build-standalone` holds write on packages'], ['actions: write beyond the audit dispatch', edit('release.yml', ' id-token: write\n attestations: write\n steps:', ' id-token: write\n attestations: write\n actions: write\n steps:'), 'build-vscode` holds write on actions'], + ['the red-team report writes contents', edit('codex-audit-report.yaml', ' contents: read\n issues: write', ' contents: write\n issues: write'), 'codex-audit-report.yaml job `publish` holds write on contents'], + ['issues: write on another job of the red-team report', edit('codex-audit-report.yaml', ' publish:\n', ' other:\n'), 'codex-audit-report.yaml job `other` holds write on issues'], ['an agent-managed job with actions: write', edit('tend-review.yaml', ' actions: read\n', ' actions: write\n'), 'tend-review.yaml job `review` holds write on actions'], ['an agent-managed job with checks: write', edit('security-audit.yaml', ' actions: read\n issues: write', ' actions: read\n checks: write\n issues: write'), 'security-audit.yaml job `audit` holds write on checks'], ['an agent-managed job with id-token: write', edit('security-audit.yaml', ' actions: read\n issues: write', ' actions: read\n issues: write\n id-token: write'), 'security-audit.yaml job `audit` holds write on id-token'], diff --git a/website/scripts/generate-canary-recall.js b/website/scripts/generate-canary-recall.js index e67c123f9..7d449b746 100644 --- a/website/scripts/generate-canary-recall.js +++ b/website/scripts/generate-canary-recall.js @@ -8,7 +8,10 @@ * runs on `main` created in the last `WINDOW_DAYS` days, downloads each one's * `canary-recall` artifact — never `canary-transcript`, which is ciphertext and * not ours to open here — validates it, and writes - * website/src/data/canary-recall.json. Run by hand with the dependency + * website/src/data/canary-recall.json. The red team's records ride along: + * each admin-dispatched `codex-audit-report.yaml` run's `codex-recall`, paired + * with the canary run whose seeds it shares (docs/specs/security-audit.md -> + * "Red team"). Run by hand with the dependency * snapshots (docs/specs/deploy.md -> "Release checklist"), because * downloading an artifact needs a token even on a public repository, and the * site is built by Cloudflare Pages, which holds none. @@ -30,6 +33,12 @@ export const ARTIFACT = 'canary-recall'; export const WINDOW_DAYS = 30; /** Every field `publicRecord` in scripts/security-audit-canary.mjs writes, and no other. */ export const RECORD_FIELDS = ['caught', 'run_url', 'seeded', 'unfinished']; +export const CODEX_WORKFLOW = 'codex-audit-report.yaml'; +/** `codex-audit-report.yaml`'s `run-name`. */ +export const CODEX_TITLE = 'codex-audit-report (advisory, not an audit)'; +export const CODEX_ARTIFACT = 'codex-recall'; +/** Every field `codexRecord` in scripts/security-audit-public-body.mjs writes, and no other. */ +export const CODEX_FIELDS = ['caught', 'claude_run_url', 'commit', 'either', 'model', 'run_url', 'seeded', 'unfinished']; const DAY = 86_400_000; const runUrl = (id) => `https://github.com/${REPO}/actions/runs/${id}`; @@ -55,9 +64,53 @@ export function validateRecord(record, id) { return { seeded: record.seeded, caught: record.caught, unfinished: record.unfinished }; } +/** + * One red-team run's `codex-recall.json`, checked as `validateRecord` checks a + * canary's: exactly the fields the builder writes, counts with caught no more + * than either and either no more than seeded, and links to this very run and + * to a run on this repository. Returns the counts and the paired run's URL. + */ +export function validateCodexRecord(record, id) { + const where = `run ${id}'s ${CODEX_ARTIFACT}`; + if (!record || typeof record !== 'object' || Array.isArray(record)) throw new Error(`${where} is not an object`); + const keys = Object.keys(record).sort(); + if (keys.join() !== CODEX_FIELDS.join()) throw new Error(`${where} has fields ${keys.join(', ')}, not ${CODEX_FIELDS.join(', ')}`); + for (const field of ['seeded', 'caught', 'either', 'unfinished']) { + if (!isCount(record[field])) throw new Error(`${where}: ${field} is not a non-negative integer`); + } + if (record.caught > record.either || record.either > record.seeded) throw new Error(`${where}: caught ${record.caught}, either ${record.either}, seeded ${record.seeded}`); + if (record.unfinished > CANARY_DOMAINS.length) throw new Error(`${where}: ${record.unfinished} unfinished domains of ${CANARY_DOMAINS.length}`); + if (record.run_url !== runUrl(id)) throw new Error(`${where} links ${JSON.stringify(record.run_url)}, not ${runUrl(id)}`); + const paired = /^https:\/\/github\.com\/diffplug\/dormouse\/actions\/runs\/([1-9]\d*)$/.exec(record.claude_run_url); + if (!paired || record.claude_run_url !== runUrl(paired[1])) throw new Error(`${where} pairs with ${JSON.stringify(record.claude_run_url)}, not a run on ${REPO}`); + return { url: record.run_url, claudeUrl: record.claude_run_url, seeded: record.seeded, caught: record.caught, either: record.either, unfinished: record.unfinished }; +} + +/** + * Each recorded canary run's red-team counts, by the canary run's URL: the + * newest red-team run paired with it. Throws on a pairing that cannot follow + * from the canary's own record — other seeds, or an `either` below Claude's + * count or above the two engines' sum. + */ +function pairCodex(recorded, codex) { + const byUrl = new Map(); + for (const c of [...codex].sort((a, b) => a.id - b.id)) { + const counts = validateCodexRecord(c.record, c.id); + const claude = recorded.find((r) => r.url === counts.claudeUrl); + if (!claude) continue; + if (counts.seeded !== claude.seeded || counts.either < claude.caught || counts.either > claude.caught + counts.caught) { + throw new Error(`run ${c.id}'s ${CODEX_ARTIFACT} cannot pair with ${claude.url}: seeded ${counts.seeded} of ${claude.seeded}, either ${counts.either} beside Claude's ${claude.caught}`); + } + const { claudeUrl: _claudeUrl, seeded: _seeded, ...kept } = counts; + byUrl.set(claude.url, kept); + } + return byUrl; +} + /** * The snapshot, from every canary run listed: `{ id, createdAt, record }`, - * `record` null for a run that left no `canary-recall` artifact. + * `record` null for a run that left no `canary-recall` artifact; and every + * red-team run listed, `{ id, record }`, each record a `codex-recall`. * * Every run in the window counts as its record has it. A domain that left no * finished report caught only what its partial report cites, so the seeds @@ -66,13 +119,15 @@ export function validateRecord(record, id) { * record at all — its seeding failed, so nothing was scored — has no counts to * add and is listed under `unrecorded`. */ -export function summarize(runs, { now, windowDays = WINDOW_DAYS }) { +export function summarize(runs, { now, windowDays = WINDOW_DAYS, codex = [] }) { const since = now.getTime() - windowDays * DAY; const inWindow = runs .filter((r) => Date.parse(r.createdAt) >= since && Date.parse(r.createdAt) <= now.getTime()) .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt) || b.id - a.id); const recorded = inWindow.filter((r) => r.record !== null) .map((r) => ({ at: r.createdAt, url: runUrl(r.id), ...validateRecord(r.record, r.id) })); + const paired = pairCodex(recorded, codex); + for (const run of recorded) if (paired.has(run.url)) run.codex = paired.get(run.url); return { asOf: now.toISOString().slice(0, 10), windowDays, @@ -100,16 +155,33 @@ function listCanaryRuns(since) { && r.head_repository?.full_name === REPO && ['schedule', 'workflow_dispatch'].includes(r.event)); } -/** The run's `canary-recall.json`, or null when the run has no unexpired one. */ -function readRecord(id) { +/** + * The successful red-team runs on `main` created since `since`, each + * dispatched by an admin: `codex-audit-report.yaml` refuses anyone else, and + * this reads the role again rather than trust that it did. + */ +function listCodexRuns(since) { + const query = `repos/${REPO}/actions/workflows/${CODEX_WORKFLOW}/runs?branch=main&status=success&created=%3E%3D${since}&per_page=100`; + const runs = gh(['api', '--paginate', '--jq', '.workflow_runs[]', query]).trim().split('\n').filter(Boolean).map((l) => JSON.parse(l)) + .filter((r) => r.display_title === CODEX_TITLE && r.head_branch === 'main' && r.head_repository?.full_name === REPO && r.event === 'workflow_dispatch'); + const roles = new Map(); + const role = (login) => { + if (!roles.has(login)) roles.set(login, gh(['api', `repos/${REPO}/collaborators/${login}/permission`, '--jq', '.role_name']).trim()); + return roles.get(login); + }; + return runs.filter((r) => /^[A-Za-z0-9-]+$/.test(r.triggering_actor?.login ?? '') && role(r.triggering_actor.login) === 'admin'); +} + +/** The run's one-file artifact `name`, parsed, or null when the run has no unexpired one. */ +function readRecord(id, name = ARTIFACT) { const artifacts = JSON.parse(gh(['api', `repos/${REPO}/actions/runs/${id}/artifacts?per_page=100`])).artifacts ?? []; - if (!artifacts.some((a) => a.name === ARTIFACT && !a.expired)) return null; - const dir = mkdtempSync(join(tmpdir(), 'canary-recall-')); + if (!artifacts.some((a) => a.name === name && !a.expired)) return null; + const dir = mkdtempSync(join(tmpdir(), `${name}-`)); try { - gh(['run', 'download', String(id), '-R', REPO, '-n', ARTIFACT, '-D', dir]); + gh(['run', 'download', String(id), '-R', REPO, '-n', name, '-D', dir]); const files = readdirSync(dir); - if (files.join() !== 'canary-recall.json') throw new Error(`run ${id}'s ${ARTIFACT} holds ${files.join(', ')}`); - return JSON.parse(readFileSync(join(dir, 'canary-recall.json'), 'utf8')); + if (files.join() !== `${name}.json`) throw new Error(`run ${id}'s ${name} holds ${files.join(', ')}`); + return JSON.parse(readFileSync(join(dir, `${name}.json`), 'utf8')); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -119,11 +191,13 @@ function main() { const now = new Date(); const since = new Date(now.getTime() - WINDOW_DAYS * DAY).toISOString().slice(0, 10); const runs = listCanaryRuns(since).map((r) => ({ id: r.id, createdAt: r.created_at, record: readRecord(r.id) })); - const snapshot = summarize(runs, { now }); + const codex = listCodexRuns(since).map((r) => ({ id: r.id, record: readRecord(r.id, CODEX_ARTIFACT) })).filter((r) => r.record !== null); + const snapshot = summarize(runs, { now, codex }); const out = resolve(dirname(fileURLToPath(import.meta.url)), '../src/data/canary-recall.json'); writeFileSync(out, `${JSON.stringify(snapshot, null, 2)}\n`); console.log(`canary recall: ${snapshot.caught}/${snapshot.seeded} over ${snapshot.runs.length} run(s) since ${since}` + - `${snapshot.unrecorded.length ? `; ${snapshot.unrecorded.length} run(s) left no record` : ''} -> ${out}`); + `${snapshot.unrecorded.length ? `; ${snapshot.unrecorded.length} run(s) left no record` : ''}` + + `; ${snapshot.runs.filter((r) => r.codex).length} paired with the red team -> ${out}`); } if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/website/scripts/generate-canary-recall.test.js b/website/scripts/generate-canary-recall.test.js index 752605f9e..1f142f405 100644 --- a/website/scripts/generate-canary-recall.test.js +++ b/website/scripts/generate-canary-recall.test.js @@ -2,7 +2,8 @@ import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { publicRecord } from '../../scripts/security-audit-canary.mjs'; -import { CANARY_TITLE, RECORD_FIELDS, WINDOW_DAYS, summarize, validateRecord } from './generate-canary-recall.js'; +import { codexRecord } from '../../scripts/security-audit-public-body.mjs'; +import { CANARY_TITLE, CODEX_FIELDS, CODEX_TITLE, RECORD_FIELDS, WINDOW_DAYS, summarize, validateCodexRecord, validateRecord } from './generate-canary-recall.js'; // docs/specs/security-audit.md -> "Canary recall". @@ -67,6 +68,53 @@ describe('summarize', () => { }); }); +const codex = (id, paired, fields = {}) => ({ claude_run_url: URL(paired), commit: 'a'.repeat(40), model: 'gpt-6-sol', + seeded: 8, caught: 5, either: 7, unfinished: 0, run_url: URL(id), ...fields }); + +describe('validateCodexRecord', () => { + it('reads exactly the fields the red-team builder writes', () => { + const written = codexRecord({ runUrl: URL(9), commit: 'a'.repeat(40), model: 'gpt-6-sol', repo: 'diffplug/dormouse', + canary: { run: '7', seeded: 8, caught: 5, either: 7, unfinished: 0 } }); + expect(Object.keys(written).sort()).toEqual(CODEX_FIELDS); + expect(validateCodexRecord(written, 9)).toEqual({ url: URL(9), claudeUrl: URL(7), seeded: 8, caught: 5, either: 7, unfinished: 0 }); + }); + + it('rejects a record out of shape or linked elsewhere', () => { + for (const [bad, message] of [ + [{ ...codex(9, 7), note: 'x' }, /has fields/], + [codex(9, 7, { either: 4 }), /caught 5, either 4/], + [codex(9, 7, { either: 9 }), /either 9, seeded 8/], + [codex(9, 7, { unfinished: 5 }), /5 unfinished/], + [codex(9, 7, { run_url: URL(8) }), /links/], + [codex(9, 7, { claude_run_url: 'https://github.com/evil/dormouse/actions/runs/7' }), /pairs with/], + ]) expect(() => validateCodexRecord(bad, 9), JSON.stringify(bad)).toThrow(message); + }); +}); + +describe('summarize with the red team', () => { + it('pairs each canary run with its newest red-team record, and ignores one paired outside the window', () => { + const snapshot = summarize([{ id: 7, createdAt: daysAgo(3), record: record(7) }], { now: NOW, codex: [ + { id: 9, record: codex(9, 7, { caught: 2, either: 6 }) }, + { id: 11, record: codex(11, 7) }, + { id: 12, record: codex(12, 1) }, + ] }); + expect(snapshot.runs).toEqual([{ at: daysAgo(3), url: URL(7), seeded: 8, caught: 6, unfinished: 0, + codex: { url: URL(11), caught: 5, either: 7, unfinished: 0 } }]); + }); + + it('refuses a pairing its canary record cannot support', () => { + const run = [{ id: 7, createdAt: daysAgo(3), record: record(7) }]; + for (const fields of [{ seeded: 6, either: 6 }, { either: 5 }, { caught: 0, either: 7 }]) { + expect(() => summarize(run, { now: NOW, codex: [{ id: 9, record: codex(9, 7, fields) }] }), JSON.stringify(fields)).toThrow(/cannot pair/); + } + }); +}); + +it('selects red-team runs by the title their workflow gives them', () => { + const workflow = readFileSync(resolve(import.meta.dirname, '../../.github/workflows/codex-audit-report.yaml'), 'utf8'); + expect(workflow).toMatch(new RegExp(`^run-name: ${CODEX_TITLE.replace(/[()]/g, '\\$&')}$`, 'm')); +}); + it('selects runs by the title the workflow gives a canary', () => { const workflow = readFileSync(resolve(import.meta.dirname, '../../.github/workflows/security-audit.yaml'), 'utf8'); expect(workflow).toMatch(new RegExp(`^run-name: .* && '${CANARY_TITLE.replace(/[()]/g, '\\$&')}' \\|\\| 'security-audit' \\}\\}$`, 'm')); diff --git a/website/src/components/CanaryRecall.test.tsx b/website/src/components/CanaryRecall.test.tsx index 7c77db64a..1b2d7117a 100644 --- a/website/src/components/CanaryRecall.test.tsx +++ b/website/src/components/CanaryRecall.test.tsx @@ -38,6 +38,18 @@ const SNAPSHOT: CanaryRecallSnapshot = { { at: "2026-09-30T00:00:00Z", url: RUN(6) }, ], }; +// The red team paired with runs 2 and 3: Oct 5's week is all paired, Oct 12's +// in part, so its either line adds run 5's Claude catches alone. +const RED_TEAM: CanaryRecallSnapshot = { + ...SNAPSHOT, + runs: SNAPSHOT.runs.map((run) => + run.url === RUN(2) + ? { ...run, codex: { url: RUN(12), caught: 5, either: 8, unfinished: 0 } } + : run.url === RUN(3) + ? { ...run, codex: { url: RUN(13), caught: 4, either: 7, unfinished: 1 } } + : run, + ), +}; const text = (html: string) => html.replace(/<[^>]+>/g, " ").replace(/'/g, "'").replace(/\s+/g, " "); /** The slot `x` puts week `i` of `n` at, as a percentage across the plot. */ const slot = (i: number, n: number) => ((i + 0.5) / n) * 100; @@ -95,6 +107,43 @@ describe("recallWeeks", () => { }); }); +describe("the red team", () => { + it("scores Codex over its paired runs, and either over every counted run", () => { + const week = Object.fromEntries(recallWeeks(RED_TEAM).map((w) => [w.start, w])); + expect(week["2026-10-05"].codex).toEqual({ runs: 1, seeded: 8, caught: 5, percent: 63, either: 8, eitherPercent: 100 }); + // Run 5 had no red team: its 8 Claude catches join run 3's 7 caught by either. + expect(week["2026-10-12"].codex).toEqual({ runs: 1, seeded: 8, caught: 4, percent: 50, either: 15, eitherPercent: 94 }); + expect(week["2026-09-21"].codex).toBeNull(); + for (const w of recallWeeks(RED_TEAM)) if (w.codex) expect(w.codex.eitherPercent).toBeGreaterThanOrEqual(w.percent!); + }); + + it("says all three in each description and run", () => { + const week = Object.fromEntries(recallWeeks(RED_TEAM).map((w) => [w.start, w])); + expect(weekDescription(week["2026-10-05"])).toBe( + "Week of Oct 5: caught 7 of 8 over 1 run (88%); Codex caught 5 of 8 over 1 run (63%); caught by either: 8 of 8 (100%)", + ); + expect(week["2026-10-12"].runs.map(runLabel)).toEqual(["Oct 12: 6 of 8*; Codex 4*, either 7", "Oct 14: no record", "Oct 16: 8 of 8"]); + }); + + it("draws three lines under a legend only once the red team has a point", () => { + const html = renderToStaticMarkup(); + const shown = text(html); + for (const label of ["Claude, the release gate", "Codex, advisory", "Caught by either"]) expect(shown).toContain(label); + expect(shown).toContain("its line covers only those runs"); + // Either and Codex span Oct 5 and Oct 12; Claude keeps its own two segments. + const lines = [...html.matchAll(/ points); + expect(lines).toEqual([ + `${slot(2, 4)},0 ${slot(3, 4)},6`, + `${slot(2, 4)},37 ${slot(3, 4)},50`, + `${slot(0, 4)},50`, + `${slot(2, 4)},12 ${slot(3, 4)},12`, + ]); + expect(html).toContain('stroke-dasharray="5 4"'); + const plain = text(renderToStaticMarkup()); + expect(plain).not.toContain("Caught by either"); + }); +}); + describe("week labels", () => { it("say counts in words, never with a slash, and list each run", () => { const week = Object.fromEntries(recallWeeks(SNAPSHOT).map((w) => [w.start, w])); @@ -135,7 +184,7 @@ describe("CanaryRecall", () => { for (const label of ["3 of 6", "no record", "7 of 8", "14 of 16*"]) expect(shown).toContain(label); expect(shown).not.toMatch(/\b\d+\/\d+\b/); for (const tick of ["100%", "50%", "0%", "Sep 21", "Oct 12"]) expect(shown).toContain(tick); - expect(shown).toContain("Hollow: a run that week had a domain"); + expect(shown).toContain("a hollow point marks a week where a Claude run had one"); expect(shown).toContain("left no record and adds nothing to its week"); expect(html).toContain(`href="${BASE.runsUrl.replace(/&/g, "&")}"`); }); diff --git a/website/src/components/CanaryRecall.tsx b/website/src/components/CanaryRecall.tsx index 8b2dfb918..e90c889fc 100644 --- a/website/src/components/CanaryRecall.tsx +++ b/website/src/components/CanaryRecall.tsx @@ -7,12 +7,22 @@ export type CanaryRecallSnapshot = { runsUrl: string; seeded: number; caught: number; - runs: { at: string; url: string; seeded: number; caught: number; unfinished: number }[]; + runs: { at: string; url: string; seeded: number; caught: number; unfinished: number; codex?: CodexCounts }[]; unrecorded: { at: string; url: string }[]; }; +/** The red team's counts over one canary run's seeds (docs/specs/security-audit.md -> "Red team"). */ +export type CodexCounts = { url: string; caught: number; either: number; unfinished: number }; + /** One run inside a week. `seeded` and `caught` are null for a run that left no record. */ -export type RecallRun = { url: string; at: string; seeded: number | null; caught: number | null; unfinished: number }; +export type RecallRun = { + url: string; + at: string; + seeded: number | null; + caught: number | null; + unfinished: number; + codex?: CodexCounts; +}; /** * One week of the window: Monday 00:00 UTC through Sunday, so the weekly @@ -30,6 +40,12 @@ export type RecallWeek = { unfinished: number; /** Runs that week that left no record, and so add nothing to its counts. */ unrecorded: number; + /** + * The red team, over that week's runs it was paired with; null when none + * was. `either` is over every counted run, a run Codex did not see adding + * Claude's catches alone, so it never falls below `percent`. + */ + codex: { runs: number; seeded: number; caught: number; percent: number; either: number; eitherPercent: number } | null; }; const DAY = 86_400_000; @@ -55,7 +71,7 @@ const plural = (n: number, word: string) => `${n} ${word}${n === 1 ? "" : "s"}`; */ export function recallWeeks(snapshot: CanaryRecallSnapshot): RecallWeek[] { const runs: RecallRun[] = [ - ...snapshot.runs.map(({ url, at, seeded, caught, unfinished }) => ({ url, at, seeded, caught, unfinished })), + ...snapshot.runs.map(({ url, at, seeded, caught, unfinished, codex }) => ({ url, at, seeded, caught, unfinished, codex })), ...snapshot.unrecorded.map(({ url, at }) => ({ url, at, seeded: null, caught: null, unfinished: 0 })), ].sort((a, b) => a.at.localeCompare(b.at) || a.url.localeCompare(b.url)); const asOf = Date.parse(`${snapshot.asOf}T00:00:00Z`); @@ -68,6 +84,10 @@ export function recallWeeks(snapshot: CanaryRecallSnapshot): RecallWeek[] { const counted = mine.filter((run) => run.seeded !== null); const seeded = counted.reduce((n, run) => n + run.seeded!, 0); const caught = counted.reduce((n, run) => n + run.caught!, 0); + const paired = counted.filter((run) => run.codex); + const codexSeeded = paired.reduce((n, run) => n + run.seeded!, 0); + const codexCaught = paired.reduce((n, run) => n + run.codex!.caught, 0); + const either = counted.reduce((n, run) => n + (run.codex ? run.codex.either : run.caught!), 0); weeks.push({ start: isoDay(monday), runs: mine, @@ -76,6 +96,16 @@ export function recallWeeks(snapshot: CanaryRecallSnapshot): RecallWeek[] { percent: seeded > 0 ? Math.round((caught / seeded) * 100) : null, unfinished: mine.filter((run) => run.unfinished > 0).length, unrecorded: mine.length - counted.length, + codex: codexSeeded > 0 + ? { + runs: paired.length, + seeded: codexSeeded, + caught: codexCaught, + percent: Math.round((codexCaught / codexSeeded) * 100), + either, + eitherPercent: Math.round((either / seeded) * 100), + } + : null, }); } return weeks; @@ -95,6 +125,12 @@ export function weekDescription(week: RecallWeek): string { ? `Week of ${dayOf(week.start)}: no counts` : `Week of ${dayOf(week.start)}: caught ${week.caught} of ${week.seeded} over ${plural(counted, "run")} (${week.percent}%)`, ]; + if (week.codex) { + parts.push( + `Codex caught ${week.codex.caught} of ${week.codex.seeded} over ${plural(week.codex.runs, "run")} (${week.codex.percent}%)`, + `caught by either: ${week.codex.either} of ${week.seeded} (${week.codex.eitherPercent}%)`, + ); + } if (week.unfinished > 0) parts.push(`${plural(week.unfinished, "run")} with a domain that left no finished report`); if (week.unrecorded > 0) parts.push(`${plural(week.unrecorded, "run")} left no record`); return parts.join("; "); @@ -103,7 +139,8 @@ export function weekDescription(week: RecallWeek): string { /** One run in a week's list, in words. */ export function runLabel(run: RecallRun): string { const counts = run.seeded === null ? "no record" : `${run.caught} of ${run.seeded}${run.unfinished > 0 ? "*" : ""}`; - return `${dayOf(run.at)}: ${counts}`; + const codex = run.codex ? `; Codex ${run.codex.caught}${run.codex.unfinished > 0 ? "*" : ""}, either ${run.codex.either}` : ""; + return `${dayOf(run.at)}: ${counts}${codex}`; } /** @@ -143,18 +180,74 @@ export function weekTicks(weeks: RecallWeek[]): { start: string; index: number } const EXTERNAL = { target: "_blank", rel: "noopener noreferrer" } as const; const GRID = [100, 50, 0]; +/** + * The three lines, told apart by dash and marker shape as well as color, since + * the color comes from the reader's theme: Claude solid in the accent with + * round points, Codex solid in the text color with square ones, either dashed + * in the accent with diamonds. + */ +const SERIES = { + claude: { label: "Claude, the release gate", stroke: "var(--docs-accent)", dash: undefined }, + codex: { label: "Codex, advisory", stroke: "color-mix(in srgb, var(--color-text) 65%, transparent)", dash: undefined }, + either: { label: "Caught by either", stroke: "var(--docs-accent)", dash: "5 4" }, +} as const; +type Series = keyof typeof SERIES; + +/** One polyline per unbroken run of weeks with a value: a week without one breaks the line. */ +function segmentsOf(values: (number | null)[], x: (i: number) => number): string[][] { + const segments: string[][] = []; + values.forEach((value, i) => { + if (value === null) return; + const point = `${x(i)},${100 - value}`; + if (i > 0 && values[i - 1] !== null) segments[segments.length - 1].push(point); + else segments.push([point]); + }); + return segments; +} + +function SeriesSwatch({ series }: { series: Series }) { + const { stroke, dash } = SERIES[series]; + return ( + + ); +} + +/** A point's shape around (cx, cy), `r` its half-width. */ +function SeriesMarker({ series, cx, cy, r }: { series: Series; cx: number; cy: number; r: number }) { + const fill = SERIES[series].stroke; + if (series === "claude") return ; + // The square a little smaller and the diamond a little larger, so the three read as the same weight. + if (series === "codex") return ; + const d = r * 0.8; + return ; +} + +/** The legend, shown once any week has a red-team point: identity is never color alone. */ +function RecallLegend() { + return ( +
    + {(Object.keys(SERIES) as Series[]).map((series) => ( +
  • + + {SERIES[series].label} +
    • + ))} +
+ ); +} + /** Percent caught per UTC week, 0–100% up the y axis, oldest to newest left to right. */ function RecallGraph({ weeks }: { weeks: RecallWeek[] }) { const n = weeks.length; const x = (i: number) => ((i + 0.5) / n) * 100; - // One polyline per unbroken run of weeks with counts: a week without breaks the line. - const segments: string[][] = []; - weeks.forEach((week, i) => { - if (week.percent === null) return; - const point = `${x(i)},${100 - week.percent}`; - if (i > 0 && weeks[i - 1].percent !== null) segments[segments.length - 1].push(point); - else segments.push([point]); - }); + const lines: [Series, string[][]][] = [ + ["either", segmentsOf(weeks.map((w) => w.codex?.eitherPercent ?? null), x)], + ["codex", segmentsOf(weeks.map((w) => w.codex?.percent ?? null), x)], + ["claude", segmentsOf(weeks.map((w) => w.percent), x)], + ]; const above = labelsAbove(weeks); return (
@@ -180,18 +273,37 @@ function RecallGraph({ weeks }: { weeks: RecallWeek[] }) { viewBox="0 0 100 100" preserveAspectRatio="none" > - {segments.map((segment) => ( - - ))} + {lines.flatMap(([series, segments]) => + segments.map((segment) => ( + + )), + )} + {/* The red team's points: the week's own point below opens its runs and describes all three. */} + {weeks.flatMap((week, i) => + week.codex + ? (["codex", "either"] as const).map((series) => ( + + )) + : [], + )}
    {weeks.map((week, i) => { const label = weekLabel(week); @@ -269,7 +381,8 @@ export default function CanaryRecall({ snapshot }: { snapshot: CanaryRecallSnaps const { asOf, windowDays, runsUrl, seeded, caught } = snapshot; const weeks = recallWeeks(snapshot); const any = snapshot.runs.length + snapshot.unrecorded.length > 0; - const unfinished = snapshot.runs.some((run) => run.unfinished > 0); + const unfinished = snapshot.runs.some((run) => run.unfinished > 0 || (run.codex?.unfinished ?? 0) > 0); + const redTeam = weeks.some((week) => week.codex !== null); return (