From 8e9b9528c5c417bac468d476ff9d5a7fa6964d35 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Tue, 20 Oct 2026 20:18:04 -0700 Subject: [PATCH 1/5] Require every Hosted Relay session to name its passkey Migration 010 deletes the sessions minted before 009, which name no passkey, and makes dormouse_relay_sessions."credentialId" NOT NULL. mintSession, the one writer, already sets it. Co-Authored-By: Claude Opus 5.5 --- docs/specs/hosted.md | 2 +- docs/specs/security-hosted.md | 4 +- .../010_relay_session_credential_required.sql | 9 ++++ hosted/server/tests/migrations.test.ts | 42 ++++++++++++++++++- hosted/server/tests/relay.test.ts | 13 ++++-- 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 hosted/server/dormouse-migrations/010_relay_session_credential_required.sql diff --git a/docs/specs/hosted.md b/docs/specs/hosted.md index 40fde3165..6f8cab974 100644 --- a/docs/specs/hosted.md +++ b/docs/specs/hosted.md @@ -229,7 +229,7 @@ Errors are the managed-voice cookie routes' ("Managed voice"), except that their - **Must count every approval attempt** against `RELAY_APPROVE_LIMIT` (10 a minute per account) before reading the body: 429 with `Retry-After` past it. - **Must answer a malformed code 400**, forgiving case, spaces, and dashes, and a second approval of a live code 409 `ALREADY_APPROVED`, whichever account sends it: a live approval never moves. An expired one is replaced. - **Must remove by deleting the Burrow's row**, its setup tokens and setup challenges cascading, so its token authenticates nothing on any relay route, **then close its live socket** ("Relay sockets"). **Never remove a passkey with its Burrow**: it is left with no Burrow (rationale). -- **Must remove a passkey by deleting its row**, every session it signed in cascading, so it signs in to nothing again. A session from before sessions named their passkey lives out its expiry. +- **Must remove a passkey by deleting its row**, every session it signed in cascading, so it signs in to nothing again; every session names the passkey that signed it in, its `"credentialId"` `NOT NULL`. - **Must take the `/enroll` fragment before render and erase it from history, and hold the code in memory only**: through email sign-in and back, never into storage. Provider sign-in leaves the page, so the user opens the link again. **Must bind every approval to its purpose, defaulting omitted purpose to remote control.** Voice approvals use a distinct code derivation and scope. Voice clients require begin to acknowledge their purpose before displaying or polling a code. Voice redemption atomically marks the approval and mints a token, without a Burrow or its cap; later polls report it spent. A remote poll cannot redeem a voice approval, or vice versa. Both purposes share approval gates and expiry. diff --git a/docs/specs/security-hosted.md b/docs/specs/security-hosted.md index 5448e3b8b..0bc6169b9 100644 --- a/docs/specs/security-hosted.md +++ b/docs/specs/security-hosted.md @@ -60,12 +60,12 @@ What ElevenLabs keeps of it is `docs/specs/security.md` -> "What is not defended | --- | --- | --- | | **Hosted's Relay keeps each account's computers, passkeys, and push registrations to that account.** It stores tokens only hashed, rechecks the owner's entitlement each time it admits a request or socket, closes a socket already open for a removed or de-entitled computer no later than the room's sweep, and neither decodes a frame nor reads a push. | Hosted phone | `hosted/server/tests/relay.test.ts`, `hosted/server/tests/relay-room.test.ts`, `hosted/server/tests/relay-push.test.ts` | -**The Hosted Relay** on the relay Worker, and its account routes on the account Worker: `docs/specs/hosted.md` -> "Relay", "Relay sockets", and "Burrow enrollment" own them; these are the checks on them. Inspect `relayApiRoutes` in `hosted/server/relay-api.ts`, `hosted/server/relay-auth.ts`, `relaySocketRoutes` in `hosted/server/relay-sockets.ts`, `RelayRoom` in `hosted/server/relay-room.ts`, the frame layer in `remote-lib-common/src/remote/relay-routing.ts`, `relayAccountRoutes` in `hosted/server/relay-account.ts`, `cookieEntitled` in `hosted/server/account-gate.ts`, `hosted/server/relay-push.ts`, and `hosted/server/dormouse-migrations/002_relay.sql`, `003_relay_push.sql`, `004_relay_enrollment_redeemed.sql`, `006_relay_setup_token_claim.sql`, `007_independent_voice.sql`, and `009_relay_passkey_removal.sql`. +**The Hosted Relay** on the relay Worker, and its account routes on the account Worker: `docs/specs/hosted.md` -> "Relay", "Relay sockets", and "Burrow enrollment" own them; these are the checks on them. Inspect `relayApiRoutes` in `hosted/server/relay-api.ts`, `hosted/server/relay-auth.ts`, `relaySocketRoutes` in `hosted/server/relay-sockets.ts`, `RelayRoom` in `hosted/server/relay-room.ts`, the frame layer in `remote-lib-common/src/remote/relay-routing.ts`, `relayAccountRoutes` in `hosted/server/relay-account.ts`, `cookieEntitled` in `hosted/server/account-gate.ts`, `hosted/server/relay-push.ts`, and `hosted/server/dormouse-migrations/002_relay.sql`, `003_relay_push.sql`, `004_relay_enrollment_redeemed.sql`, `006_relay_setup_token_claim.sql`, `007_independent_voice.sql`, `009_relay_passkey_removal.sql`, and `010_relay_session_credential_required.sql`. - **FAIL IF** `S97` a session, Burrow, or setup token is stored other than as its SHA-256, an enrollment device code is stored at all, or an account's Relay rows outlive its user row. - **FAIL IF** `S98` a query reading a Burrow, passkey, presence nonce, or setup token is not scoped to the caller's account (the session's user, the Burrow token's owner, or the setup token's Burrow's owner; `RelayRows` by its consumer's account check; sign-in excepted), a passkey registers to any account but the minting Burrow's owner, or a setup challenge redeems with another Burrow's token. - **FAIL IF** `S99` a setup token, challenge, or presence nonce is checked and spent in separate statements, so two concurrent redeemers can both win, or a retire misses a setup token a `finish` claimed, a refused `finish` gives its token a later expiry or a second use rather than clearing that claim or spending the token, or a passkey insert does not first re-read the claim `FOR UPDATE`. -- **FAIL IF** `S100` a Burrow-authenticated request, setup redemption, enrollment redemption, sign-in, or session-gated request skips rechecking the owner's entitlement (`entitledSql` on its user row) in that request, so a de-entitled account's session acts or a sign-in mints one; or a removed Burrow's row, setup tokens, or setup challenges survive its removal, or removal reaches another account's Burrow; or a removed passkey's sessions survive its removal, or removal reaches another account's passkey. +- **FAIL IF** `S100` a Burrow-authenticated request, setup redemption, enrollment redemption, sign-in, or session-gated request skips rechecking the owner's entitlement (`entitledSql` on its user row) in that request, so a de-entitled account's session acts or a sign-in mints one; or a removed Burrow's row, setup tokens, or setup challenges survive its removal, or removal reaches another account's Burrow; or a session can be stored naming no passkey, a removed passkey's sessions survive its removal, or removal reaches another account's passkey. - **FAIL IF** `S101` a route that presents no credential writes a row other than a sign-in challenge (enrollment begin writes none); setup routes write only under a live setup token and a poll only under an approval; a device code's expiry is read from anywhere but its own leading bytes, or a code past it reaches the database; or a user code is anything but the purpose-bound derivation in `docs/specs/hosted.md` -> "Burrow enrollment". - **FAIL IF** `S102` an approval is redeemed other than in the one statement that marks it redeemed and inserts only its purpose's credential (Burrow or voice token), or a redeemed approval redeems again (removing its Burrow included), so two polls can mint two Burrows; the Burrow is owned by anyone but the approval's `userId`; a poll redeems an approval for another purpose or a user code its device code does not derive; an approval replaces a live one; or a poll or approval treats an approval past its `expiresAt` as live (an approval may only replace it). - **FAIL IF** `S103` enrollment begin or poll admits a request carrying `Origin`, or either reaches the database before its per-address limit. diff --git a/hosted/server/dormouse-migrations/010_relay_session_credential_required.sql b/hosted/server/dormouse-migrations/010_relay_session_credential_required.sql new file mode 100644 index 000000000..319cd5c41 --- /dev/null +++ b/hosted/server/dormouse-migrations/010_relay_session_credential_required.sql @@ -0,0 +1,9 @@ +-- Up Migration +-- Every session names the passkey that signed it in, so removing that passkey +-- ends it (docs/specs/hosted.md -> "Burrow enrollment"). A session minted +-- before 009 names none and is deleted here; its holder signs in again. +DELETE FROM dormouse_relay_sessions WHERE "credentialId" IS NULL; +ALTER TABLE dormouse_relay_sessions ALTER COLUMN "credentialId" SET NOT NULL; + +-- Down Migration +ALTER TABLE dormouse_relay_sessions ALTER COLUMN "credentialId" DROP NOT NULL; diff --git a/hosted/server/tests/migrations.test.ts b/hosted/server/tests/migrations.test.ts index 9ec2e01c8..09806ad75 100644 --- a/hosted/server/tests/migrations.test.ts +++ b/hosted/server/tests/migrations.test.ts @@ -1,6 +1,13 @@ import { test, expect } from "vitest"; -import { createHash } from "node:crypto"; -import { readdirSync, readFileSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { copyFileSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { betterAuthMigrations } from "@pgstencil/auth/better-auth-migrations"; +import { migrate, readMigrations } from "pgstencil/database"; +import { queryDatabase } from "pgstencil/postgres"; +import { createTestContext } from "pgstencil/testing"; /** * Every Dormouse migration, by the SHA-256 of its LF-normalized text @@ -24,6 +31,8 @@ const PINNED: Record = { "b3f265e4c99b7d78c84aa58fe3e95b9a609c450ccc0a7f3701f871c8798d4801", "009_relay_passkey_removal.sql": "c6c220960187e33a1350dc85e93d24f99866a72a579ed9def6a3663643f3e928", + "010_relay_session_credential_required.sql": + "f385398d16126381b6a976d74e23e10b07370c1a1da3205e71697aa62a3399f5", }; const directory = new URL("../dormouse-migrations/", import.meta.url); @@ -37,3 +46,32 @@ test("no merged Dormouse migration changes, and every one is pinned", () => { expect(digest, `${name} was edited; append a new migration instead`).toBe(PINNED[name]); } }); + +test("010 deletes the sessions that name no passkey, and every later one must name one", async ({ + onTestFinished, +}) => { + // The Dormouse migrations through 009, then 010 over the rows they allowed. + const before = mkdtempSync(join(tmpdir(), "dormouse-migrations-")); + onTestFinished(() => rmSync(before, { recursive: true, force: true })); + const source = fileURLToPath(directory); + for (const name of Object.keys(PINNED).filter((name) => name < "010")) copyFileSync(join(source, name), join(before, name)); + const context = await createTestContext({ migrations: [betterAuthMigrations, before] }); + onTestFinished(context.close); + const sql = (text: string, values?: unknown[]) => queryDatabase(context.database.url, text, values); + const userId = randomUUID(); + await sql(`INSERT INTO "user" (id, name, email, "emailVerified") VALUES ($1, $1, $2, true)`, [userId, `${userId}@example.test`]); + await sql(`INSERT INTO dormouse_relay_passkeys ("credentialId", "userId", "publicKey", label) VALUES ('pk', $1, 'x', '')`, [userId]); + const session = (tokenHash: string, credentialId: string | null) => + sql( + `INSERT INTO dormouse_relay_sessions ("tokenHash", "userId", "expiresAt", "credentialId") + VALUES ($1, $2, now() + interval '1 hour', $3)`, + [tokenHash, userId, credentialId], + ); + await session("named", "pk"); + await session("unnamed", null); + + copyFileSync(join(source, "010_relay_session_credential_required.sql"), join(before, "010_relay_session_credential_required.sql")); + await migrate(context.database.url, await readMigrations([betterAuthMigrations, before])); + expect(await sql(`SELECT "tokenHash" FROM dormouse_relay_sessions`)).toEqual([{ tokenHash: "named" }]); + await expect(session("later", null)).rejects.toThrow(/credentialId/); +}); diff --git a/hosted/server/tests/relay.test.ts b/hosted/server/tests/relay.test.ts index cedf9afcb..09157c550 100644 --- a/hosted/server/tests/relay.test.ts +++ b/hosted/server/tests/relay.test.ts @@ -1146,13 +1146,18 @@ test("begin refuses another origin", async ({ onTestFinished }) => { const FCM = "https://fcm.googleapis.com/fcm/send/"; -/** A live session for `userId`, as sign-in writes one. */ +/** A live session for `userId`, signed in by a passkey of its own, as sign-in writes one. */ async function sessionFor(f: Awaited>, userId: string) { const token = randomSecret(); + const credentialId = randomSecret(); await f.sql( - `INSERT INTO dormouse_relay_sessions ("tokenHash", "userId", "expiresAt") - VALUES ($1, $2, now() + interval '1 hour')`, - [digest(token), userId], + `INSERT INTO dormouse_relay_passkeys ("credentialId", "userId", "publicKey", label) VALUES ($1, $2, 'x', '')`, + [credentialId, userId], + ); + await f.sql( + `INSERT INTO dormouse_relay_sessions ("tokenHash", "userId", "expiresAt", "credentialId") + VALUES ($1, $2, now() + interval '1 hour', $3)`, + [digest(token), userId, credentialId], ); return token; } From dd8c118afa1118e89cc7aa2ceb0f920faa2b68b4 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Tue, 20 Oct 2026 20:32:30 -0700 Subject: [PATCH 2/5] Name the Codex engine the advisory engine, apart from the red team The audit checks every FAIL IF: Claude in CI is the release gate and Codex its advisory engine. The red team is the separate quarterly pass over the code and the published guarantees without the FAIL IF lines. security-audit.md splits the two into their own sections, and the script, workflow, builder, generator, and /security copy follow. AUDIT_OPEN_FINDINGS= hands the local runner's domains the open findings from a file instead of listing them with gh, so the gh login a Codex sandbox reaches needs no read on them. A Codex canary without Claude's scorecard now prints its publish command with the either count left empty. The builder accepts it, the codex-recall record carries either: null, and the recall generator and graph read that as not computed: no point on the either line that week. Co-Authored-By: Claude Opus 5.5 --- .github/audit/ci-and-secrets.md | 2 +- .github/workflows/codex-audit-report.yaml | 18 ++--- docs/specs/security-audit.md | 22 ++++-- docs/specs/security-audit.rationale.md | 14 +++- docs/specs/security-ci.md | 2 +- docs/specs/security.md | 2 +- scripts/security-audit-canary.test.mjs | 17 +++-- scripts/security-audit-local.sh | 61 ++++++++++----- scripts/security-audit-public-body.mjs | 53 +++++++------ scripts/security-audit.test.mjs | 56 ++++++++++++-- scripts/spec-word-budgets.json | 2 +- scripts/workflow-lint.test.mjs | 4 +- website/scripts/generate-canary-recall.js | 47 ++++++------ .../scripts/generate-canary-recall.test.js | 22 +++++- website/src/components/CanaryRecall.test.tsx | 45 ++++++++--- website/src/components/CanaryRecall.tsx | 75 ++++++++++++------- 16 files changed, 298 insertions(+), 144 deletions(-) diff --git a/.github/audit/ci-and-secrets.md b/.github/audit/ci-and-secrets.md index 5c9622208..1be6c0e0e 100644 --- a/.github/audit/ci-and-secrets.md +++ b/.github/audit/ci-and-secrets.md @@ -30,7 +30,7 @@ Delegate only by these streams (`.github/audit/_preamble.md` -> "Work streams"). - `actions` — `docs/specs/security-ci.md`: "GitHub Actions Policies", "Automated Maintainer (tend)". - `releases` — `docs/specs/security-ci.md`: "Hosted Deployments", "VS Code Extension Releases", "Desktop Releases". -- `audit` — `docs/specs/security-audit.md`: "Schedule and gate", "Deterministic checks", "Skipping an unchanged audit", "Domains", "Orchestration", "Outcomes and reporting", "Findings", "Embargo", "Findings ledger", "Disclosure", "Environment and `AUDIT_PAT`", "Canary recall", "Red team". +- `audit` — `docs/specs/security-audit.md`: "Schedule and gate", "Deterministic checks", "Skipping an unchanged audit", "Domains", "Orchestration", "Outcomes and reporting", "Findings", "Embargo", "Findings ledger", "Disclosure", "Environment and `AUDIT_PAT`", "Canary recall", "Advisory engine". - `qualitative` — the qualitative pass below, once, over the whole of this domain's scope. ## Qualitative pass diff --git a/.github/workflows/codex-audit-report.yaml b/.github/workflows/codex-audit-report.yaml index a46df8d9b..f63aad78f 100644 --- a/.github/workflows/codex-audit-report.yaml +++ b/.github/workflows/codex-audit-report.yaml @@ -1,10 +1,10 @@ name: codex-audit-report -# Publishes the red team's report (docs/specs/security-audit.md -> "Red team"): -# what a maintainer's `scripts/security-audit-local.sh --engine codex` run -# computed, typed in as dispatch inputs. Nothing here runs a model or reads a -# secret, and nothing here gates a release: release.yml waits on -# security-audit.yaml alone. The inputs are non-secret by construction — +# Publishes the advisory engine's report (docs/specs/security-audit.md -> +# "Advisory engine"): what a maintainer's `scripts/security-audit-local.sh +# --engine codex` run computed, typed in as dispatch inputs. Nothing here runs +# a model or reads a secret, and nothing here gates a release: release.yml +# waits on security-audit.yaml alone. The inputs are non-secret by construction — # computed verdicts and counts, never a finding — and # `scripts/security-audit-public-body.mjs` checks each against its pattern, # and the counts against the CI canary run they share seeds with, before it @@ -38,7 +38,7 @@ on: type: string default: "" canary_either: - description: "Seeds Claude or Codex caught" + description: "Seeds Claude or Codex caught; empty when not computed" type: string default: "" canary_unfinished: @@ -65,7 +65,7 @@ jobs: - name: Refuse any ref but main run: | [ "$GITHUB_REF" = refs/heads/main ] && exit 0 - echo "::error::A red-team report publishes from main only." + echo "::error::An advisory-engine report publishes from main only." exit 1 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -79,7 +79,7 @@ jobs: # Inputs reach the builder through the environment, never interpolated # into the script. It refuses a dispatch by anyone but an admin. - - name: Publish the red-team report + - name: Publish the advisory-engine report env: GH_TOKEN: ${{ github.token }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -103,7 +103,7 @@ jobs: # The counts website/scripts/generate-canary-recall.js reads beside the # CI canary run's own `canary-recall`. - - name: Upload the red-team recall + - name: Upload the advisory-engine recall if: inputs.canary_run != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/docs/specs/security-audit.md b/docs/specs/security-audit.md index 9343dc90e..5de171db8 100644 --- a/docs/specs/security-audit.md +++ b/docs/specs/security-audit.md @@ -1,6 +1,6 @@ # Security Audit -> - Owns how the security specs are audited: the schedule and the release gate, the deterministic checks and the skip they allow, the domains and their prompts, the orchestration contract, the three outcomes, the reporting steps, the embargo on finding detail and its disclosure, the environment that holds `AUDIT_PAT`, the canary that measures recall, and the advisory Codex red team. +> - Owns how the security specs are audited: the schedule and the release gate, the deterministic checks and the skip they allow, the domains and their prompts, the orchestration contract, the three outcomes, the reporting steps, the embargo on finding detail and its disclosure, the environment that holds `AUDIT_PAT`, the canary that measures recall, the advisory Codex engine, and the red team. > - Defers what is audited to `docs/specs/security.md` and the specs it names, and each agent's procedure to its prompt in `.github/audit/`. > - Read `docs/specs/security.md` first. @@ -48,7 +48,7 @@ CI and the local runner share the prompts and their scopes in `.github/audit/` ( - **FAIL IF** `S5` a `docs/specs/security*.md` spec (not a `.rationale.md`) is in no domain's scope, or in two, or a scope names a file that does not exist (rationale). - **FAIL IF** `S6` `.github/audit/orchestrator.md` lets the orchestrator audit anything itself (rationale). - **FAIL IF** `S7` the audit stops fanning out to a dedicated `remote` subagent scoped to `docs/specs/security-remote.md`, a dedicated `local` subagent scoped to `docs/specs/security-local.md`, or a dedicated `hosted` subagent scoped to `docs/specs/security-hosted.md`, or any of those scopes is merged back into a context that also carries another domain (rationale). -- **FAIL IF** `S8` `remote`, `local`, or `hosted` does not run on Opus and the mechanical domains on Sonnet, the stronger model on the code-reading domains, in **both** `.github/workflows/security-audit.yaml`'s `claude_args` — its `--model` sets the floor and its `--agents` raises those three domains — and `scripts/security-audit-local.sh`'s default Claude engine (rationale). This split is the release gate; the Codex engine never is ([Red team](#red-team)). Pinned by `scripts/security-audit.test.mjs`. +- **FAIL IF** `S8` `remote`, `local`, or `hosted` does not run on Opus and the mechanical domains on Sonnet, the stronger model on the code-reading domains, in **both** `.github/workflows/security-audit.yaml`'s `claude_args` — its `--model` sets the floor and its `--agents` raises those three domains — and `scripts/security-audit-local.sh`'s default Claude engine (rationale). This split is the release gate; the Codex engine never is ([Advisory engine](#advisory-engine)). Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S9` `.github/audit/` is missing a prompt file the workflow names, or `scripts/security-audit-local.sh` stops running the audit from those same files (rationale), or exits zero after a failed process, a missing or unfinished fragment, or any verdict other than exact `VERDICT: PASS`. Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S10` the union of the subagents' qualitative scopes does not cover every top-level path in the repository: the catch-all's remainder stops subtracting every other domain's prompt, or a second domain claims the remainder (rationale). Pinned by `scripts/security-audit.test.mjs`. - **FAIL IF** `S11` `.github/audit/` or `.vscode/` is outside **any** consumer of `.github/workflows/workflow-audit.yaml`'s diff window — the commit list, `own_changes`, `is_clean_merge`, and both content classifiers' refusals, whose half is *derived* from the single `WINDOW` array. The security specs are deliberately *not* watched there (rationale). @@ -121,7 +121,7 @@ Source of truth: `.github/audit/_preamble.md`; `dedupFindings` in `scripts/secur | Sink | Carries | |---|---| -| Red-team comment on the status issue, one per `codex-audit-report.yaml` run | what [Red team](#red-team) lists | +| Advisory-engine comment on the status issue, one per `codex-audit-report.yaml` run | what [Advisory engine](#advisory-engine) lists | | `codex-recall` artifact | that run's canary counts, model, commit, and the two run links | | Public status issue, labeled `security-audit-status` | the run's time, link, commit, and outcome; whether the domains were skipped and when they last ran; each domain's computed verdict and its failed-check, missing-rule, malformed-line, BLOCKER, and WARNING counts; each failed check's spec and heading; how many verdict lines disagreed; whether the private filing failed; each open ledger finding's severity, kind, and dates, keyed as [Disclosure](#disclosure) allows; the latest disclosures | | Public disclosure issue, labeled `security-audit-finding`, one per ledger issue | what [Disclosure](#disclosure) lists | @@ -219,24 +219,30 @@ Source of truth: `seed` and `score` in `scripts/security-audit-canary.mjs`; the ### Published recall -dormouse.sh/security shows a rolling recall under "How the guarantees are checked": caught over seeded across the `canary-recall` artifacts of canary runs on `main` in the 30 days before its as-of date, each run linked. Its weekly graph draws three lines: Claude over every run; Codex over the runs a [Red team](#red-team) record pairs with; and caught by either, where a run no record pairs adds Claude's catches alone. It is a committed snapshot, `website/src/data/canary-recall.json`, regenerated at each release (`docs/specs/deploy.md` -> "Release checklist") (rationale). +dormouse.sh/security shows a rolling recall under "How the guarantees are checked": caught over seeded across the `canary-recall` artifacts of canary runs on `main` in the 30 days before its as-of date, each run linked. Its weekly graph draws three lines: Claude over every run; Codex over the runs an [Advisory engine](#advisory-engine) record pairs with; and caught by either, where a run no record pairs adds Claude's catches alone, and a paired run with a null either count, not computed rather than zero, leaves its week no point. It is a committed snapshot, `website/src/data/canary-recall.json`, regenerated at each release (`docs/specs/deploy.md` -> "Release checklist") (rationale). - **A record out of shape fails the snapshot**, never repaired or skipped: a field but the scorer's, a count not a non-negative integer, more caught than seeded, or a link but to its own run on `diffplug/dormouse`. - **Every run in the window counts as its record has it** (rationale): an unfinished domain's seeds stay seeded, caught only by what it reported, and the run shows its unfinished count. A run with no `canary-recall` artifact is listed without counts. - The generator downloads `canary-recall` and `codex-recall` alone, never `canary-transcript`. -- **A red-team record pairs with its canary run by link**, the newest successful run on `main` per canary run; one whose seeded count differs from that run's, or whose either count falls below Claude's or above the two engines' sum, fails the snapshot. +- **An advisory-engine record pairs with its canary run by link**, the newest successful run on `main` per canary run; one whose seeded count differs from that run's, or whose either count falls below Claude's or above the two engines' sum, fails the snapshot. Source of truth: `summarize` in `website/scripts/generate-canary-recall.js`; `CanaryRecall` in `website/src/components/CanaryRecall.tsx`. -## Red team +## Advisory engine + +**Codex runs the audit from outside its model family, and is advisory: no Codex verdict gates a release, feeds the skip, or files** (rationale). `scripts/security-audit-local.sh --engine codex` runs the same prompts on `gpt-6-sol` under the `daybreak_blue` Cyber access program, sandboxed to the working tree. A maintainer runs every domain, and a canary keyed by the latest CI canary run on `main`, which counts the seeds either engine caught given that run's decrypted scorecard, and leaves that count empty without it (rationale). A Codex BLOCKER or WARNING reaches the ledger only as a maintainer's hand-filed `manual-` key ([Findings ledger](#findings-ledger)). -**Codex audits the audit from outside its model family, and is advisory: no Codex verdict gates a release, feeds the skip, or files** (rationale). `scripts/security-audit-local.sh --engine codex` runs the same prompts on `gpt-6-sol` under the `daybreak_blue` Cyber access program, sandboxed to the working tree. **A maintainer runs it quarterly, and before any release that changes remote control or Hosted**: every domain, and a canary keyed by the latest CI canary run on `main` and scored against that run's decrypted scorecard. A Codex BLOCKER or WARNING reaches the ledger only as a maintainer's hand-filed `manual-` key ([Findings ledger](#findings-ledger)). +**On a machine running Codex, the `gh` login holds no read on the ledger**, since the sandbox reaches it (rationale): `AUDIT_OPEN_FINDINGS=` hands the domains a list written on another login, copied to `audit-open-findings.txt` in place of listing the ledger. `.github/workflows/codex-audit-report.yaml` publishes the run from `workflow_dispatch` inputs that carry nothing a finding could be read from: the commit, the model, each domain's computed verdict, and the canary's run ID and seeded, Codex-caught, either-caught, and unfinished counts. `scripts/security-audit-public-body.mjs` checks each against its pattern and the commit against `main`, checks the counts against that canary run's own record and commit, comments once on the status issue, and writes the `codex-recall` record [Published recall](#published-recall) pairs. - **FAIL IF** `S275` `.github/workflows/codex-audit-report.yaml` can gate, file, or publish past its inputs: it holds any write permission but its one job's `issues: write`, reads a secret, interpolates an input into a script, runs on a ref but `main`, posts for anyone but an admin or anything but `codexComment`'s text, or the gate in `.github/workflows/release.yml` can select its run; or `scripts/security-audit-local.sh` runs Codex unsandboxed or on another model or program. Pinned by `scripts/security-audit.test.mjs`. -Source of truth: `run_domain` in `scripts/security-audit-local.sh`; `publishCodex` in `scripts/security-audit-public-body.mjs`; `.github/workflows/codex-audit-report.yaml`. +Source of truth: `run_domain` and `write_open_findings` in `scripts/security-audit-local.sh`; `publishCodex` in `scripts/security-audit-public-body.mjs`; `.github/workflows/codex-audit-report.yaml`. + +## Red team + +**A maintainer runs a red team quarterly, and before any release that changes remote control or Hosted** (rationale): an open-ended pass over the code and the guarantees dormouse.sh/security publishes, without the `FAIL IF` lines. Its BLOCKERs and WARNINGs reach the ledger as hand-filed `manual-` keys ([Findings ledger](#findings-ledger)). ## Future diff --git a/docs/specs/security-audit.rationale.md b/docs/specs/security-audit.rationale.md index 578244b2c..7ee56c57a 100644 --- a/docs/specs/security-audit.rationale.md +++ b/docs/specs/security-audit.rationale.md @@ -202,12 +202,20 @@ gh secret set AUDIT_PAT --env security-audit --repo diffplug/dormouse --body 'gi **Why an unfinished domain still counts.** Dropping the run, or its unfinished domain's seeds, would raise the figure by the runs where the audit did worst; a domain that ran out of time is a miss the published recall should carry. -## Red team +## Advisory engine **Why a second model family (2026-10).** Every domain, the canary's included, runs on one vendor's models, so a blind spot they share is invisible to the recall figure. Codex reads the same prompts and seeds, which isolates the model as the variable: the either line is what the two together would catch. -**Why advisory.** A gate needs a run that is reproducible on the release tag, with credentials CI holds; the red team runs on a maintainer's machine and login, quarterly. Its value is a second opinion and a recall comparison, not a pass. +**Why advisory.** A gate needs a run that is reproducible on the release tag, with credentials CI holds; the advisory engine runs on a maintainer's machine and login. Its value is a second opinion and a recall comparison, not a pass. + +**Why the either count may be empty (2026-10).** Claude's scorecard sits in its canary run's encrypted transcript, which not every machine that runs Codex can decrypt. Codex's own counts stand without it; a zero in its place would plot a false either point. + +**Why a file for the open findings (2026-10).** `ci-and-secrets` reads live GitHub state, so the Codex sandbox has the network and reaches the operator's `gh` login; a login that lists the ledger would hand the model every open finding's title, and through the same login its body. **Why typed inputs rather than an uploaded report.** Anything the Codex run wrote could carry finding detail past the embargo; verdict words, a run ID, and four counts cannot, and a pattern check proves it. The counts are checked against the Claude canary's own record so a mistyped either count fails rather than publishing. -**Why quarterly, and before remote control or Hosted releases.** Those are the two surfaces an attacker reaches without local access, and where the code-reading domains run longest; a release changing them is when a miss costs most. +## Red team + +**Why quarterly, and before remote control or Hosted releases.** Those are the two surfaces an attacker reaches without local access, so a release changing them is when a miss costs most. + +**Why without the `FAIL IF` lines.** The audit and its advisory engine both read the rules, so a hole no rule names is invisible to them; a pass from the code and the published guarantees alone looks where the rules do not. diff --git a/docs/specs/security-ci.md b/docs/specs/security-ci.md index a309cc7b1..beeb99d4c 100644 --- a/docs/specs/security-ci.md +++ b/docs/specs/security-ci.md @@ -20,7 +20,7 @@ Renovate updates the commit pins; the generated `tend-*.yaml` pin `max-sixty/ten - **FAIL IF** `S42` `.github/workflows/release.yml` runs on anything but a pushed `v*` tag. Pinned by `scripts/workflow-lint.mjs`. - **FAIL IF** `S43` the `security-audit` job in `.github/workflows/release.yml` uses its `actions: write` for anything but dispatching `security-audit.yaml` on the release tag and watching that run. Pinned by `scripts/workflow-lint.mjs`. - **FAIL IF** `S44` `pull_request_target` appears as an `on:` trigger in any `.github/workflows/**` file other than `tend-*.yaml`. A mention in a comment is not a violation. Pinned by `scripts/workflow-lint.mjs`. -- **FAIL IF** `S45` a non-agent-managed workflow has **effective** write permissions other than the release provenance permissions `id-token: write` and `attestations: write`, the `actions: write` granted to the `security-audit` job in `release.yml` (see "Release audit dispatch"), or the `issues: write` granted to the `publish` job in `codex-audit-report.yaml` (`docs/specs/security-audit.md` -> "Red team"). Effective as in the agent-managed bullet below. Pinned by `scripts/workflow-lint.mjs`. +- **FAIL IF** `S45` a non-agent-managed workflow has **effective** write permissions other than the release provenance permissions `id-token: write` and `attestations: write`, the `actions: write` granted to the `security-audit` job in `release.yml` (see "Release audit dispatch"), or the `issues: write` granted to the `publish` job in `codex-audit-report.yaml` (`docs/specs/security-audit.md` -> "Advisory engine"). Effective as in the agent-managed bullet below. Pinned by `scripts/workflow-lint.mjs`. `.github/workflows/ci.yml` also runs actionlint and zizmor over every workflow; `.github/zizmor.yml` names each exception it accepts. diff --git a/docs/specs/security.md b/docs/specs/security.md index 67286a74a..6275c705d 100644 --- a/docs/specs/security.md +++ b/docs/specs/security.md @@ -38,7 +38,7 @@ Every section the [nightly audit](#how-the-guarantees-are-checked) checks opens **Every week, a canary audits the audit.** It seeds already-fixed or synthetic vulnerabilities into a throwaway checkout, some beside comments that justify them, removes the tests and lints that would catch them, runs the code domains over it, and publishes only how many they caught ([Canary recall](./security-audit.md#canary-recall)). -**Every quarter, and before releases that change remote control or Hosted, Codex red-teams the audit**: the same prompts and seeds on another vendor's model, published beside the nightly but never gating a release ([Red team](./security-audit.md#red-team)). +**Every quarter, and before releases that change remote control or Hosted, a red team reads the code and these guarantees without the `FAIL IF` lines**, for what the checks themselves miss ([Red team](./security-audit.md#red-team)). A maintainer also runs the audit's prompts and canary seeds on Codex, another vendor's model, as an advisory engine: published beside the nightly, never gating a release ([Advisory engine](./security-audit.md#advisory-engine)). Source of truth: root scripts in `package.json`; native jobs in `.github/workflows/ci.yml`; `.github/workflows/security-audit.yaml` and its release gate in `.github/workflows/release.yml`. diff --git a/scripts/security-audit-canary.test.mjs b/scripts/security-audit-canary.test.mjs index 3ab3875fa..12ffe6068 100644 --- a/scripts/security-audit-canary.test.mjs +++ b/scripts/security-audit-canary.test.mjs @@ -436,9 +436,9 @@ test('the local canary seeds a copy, runs the code domains, scores, and files no assert.ok(existsSync(join(dir, POOL_DIR, 'room.patch'))); }); -// The red team: the same seeds under a CI run's key, scored against that -// run's scorecard, and the publishing command printed, never run. -test('the local Codex canary takes a CI run\'s key, counts either engine\'s catches, and files nothing', (t) => { +// The advisory engine: the same seeds under a CI run's key, scored against +// that run's scorecard, and the publishing command printed, never run. +test('the local Codex canary takes a CI run\'s key, publishes with or without Claude\'s scorecard, and files nothing', (t) => { const { base, dir, bin, env } = checkout(t); const head = execFileSync(realGit, ['rev-parse', 'HEAD'], { cwd: dir, encoding: 'utf8' }).trim(); // A commit past the run's: the canary still seeds the run's own. @@ -468,6 +468,11 @@ test('the local Codex canary takes a CI run\'s key, counts either engine\'s catc fs.writeFileSync(output, ${JSON.stringify(fragment([]))}); `); stub(bin, 'claude', 'process.exit(5);'); + // Without Claude's scorecard, the command publishes Codex's own counts and leaves either empty. + const alone = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'codex', 'canary', '3', '--key', '42'], { cwd: dir, env, encoding: 'utf8' }); + assert.equal(alone.status, 0, alone.stderr); + assert.doesNotMatch(alone.stdout, /^either:/m); + assert.ok(alone.stdout.includes(`gh workflow run codex-audit-report.yaml -R fixture/repo --ref main -f commit=${head} -f model=gpt-6-sol -f canary_run=42 -f canary_seeded=2 -f canary_caught=0 -f canary_unfinished=0\n`), alone.stdout); const result = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'codex', 'canary', '3', '--key', '42', '--claude-scorecard', claudeCard], { cwd: dir, env, encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); @@ -475,10 +480,10 @@ test('the local Codex canary takes a CI run\'s key, counts either engine\'s catc assert.match(result.stdout, /^either: 1\/2 seeds caught by this engine or the other$/m); assert.ok(result.stdout.includes(`gh workflow run codex-audit-report.yaml -R fixture/repo --ref main -f commit=${head} -f model=gpt-6-sol -f canary_run=42 -f canary_seeded=2 -f canary_caught=0 -f canary_either=1 -f canary_unfinished=0\n`), result.stdout); const codexCalls = calls(base).filter((c) => c.startsWith('codex ')); - assert.equal(codexCalls.length, CANARY_DOMAINS.length); + assert.equal(codexCalls.length, 2 * CANARY_DOMAINS.length); for (const c of codexCalls) assert.equal(c, 'codex exec -m gpt-6-sol --cyber-access-program daybreak_blue --sandbox workspace-write -c sandbox_workspace_write.network_access=true --ephemeral'); - // One read, the run's commit, on each engine's run; nothing filed. - assert.deepEqual(calls(base).filter((c) => c.startsWith('gh ')), Array(2).fill('gh api repos/fixture/repo/actions/runs/42 --jq .head_sha')); + // One read, the run's commit, on each run; nothing filed. + assert.deepEqual(calls(base).filter((c) => c.startsWith('gh ')), Array(3).fill('gh api repos/fixture/repo/actions/runs/42 --jq .head_sha')); }); test('a union counts each seed once, and only over the same seeds', () => { diff --git a/scripts/security-audit-local.sh b/scripts/security-audit-local.sh index f4a01b0fc..90f7ca5dc 100755 --- a/scripts/security-audit-local.sh +++ b/scripts/security-audit-local.sh @@ -25,13 +25,20 @@ # `--key` takes a CI canary run's seed key, its run ID: the canary then seeds # that run's commit, read on your `gh` login, with the seeds that run dealt. # -# `--engine codex` runs the same prompts on Codex instead of Claude: the red -# team (docs/specs/security-audit.md -> "Red team"), advisory, never the -# release gate. Its canary takes `--claude-scorecard`, the decrypted +# `--engine codex` runs the same prompts on Codex instead of Claude: the +# advisory engine (docs/specs/security-audit.md -> "Advisory engine"), never +# the release gate. Its canary takes `--claude-scorecard`, the decrypted # `scorecard.json` of the CI canary run its `--key` names, to count the seeds -# either engine caught. It ends by printing the `gh workflow run -# codex-audit-report.yaml` command that publishes its verdicts and counts; -# running that is yours to do, since this script files nothing. +# either engine caught; without it that count is left empty. It ends by +# printing the `gh workflow run codex-audit-report.yaml` command that +# publishes its verdicts and counts; running that is yours to do, since this +# script files nothing. +# +# `AUDIT_OPEN_FINDINGS=` hands every domain the open findings from that +# file, one ledger title per line as CI's `List open findings` writes them, +# copied to `audit-open-findings.txt` in place of listing the ledger with +# `gh`. A Codex run's sandbox reaches this run's `gh` login, so on its machine +# that login must not read the ledger: write the list on another. # # The deterministic GitHub-state check (`scripts/github-state-check.mjs`) runs # first whenever a domain that reads its fragment does — `ci-and-secrets` and @@ -42,6 +49,12 @@ set -euo pipefail +# Resolved before the `cd` below, so a relative path is the operator's. +if [ -n "${AUDIT_OPEN_FINDINGS:-}" ]; then + [ -f "$AUDIT_OPEN_FINDINGS" ] || { echo "error: AUDIT_OPEN_FINDINGS names no file: $AUDIT_OPEN_FINDINGS" >&2; exit 64; } + AUDIT_OPEN_FINDINGS="$(cd "$(dirname "$AUDIT_OPEN_FINDINGS")" && pwd)/$(basename "$AUDIT_OPEN_FINDINGS")" +fi + cd "$(dirname "$0")/.." AUDIT_DIR=.github/audit export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-diffplug/dormouse}" @@ -63,8 +76,8 @@ case "$ENGINE" in claude|codex) ;; *) echo "error: unknown engine '$ENGINE' (claude|codex)" >&2; exit 64 ;; esac -# The red team's model and Cyber access program. Pinned here alone: the -# Codex engine has no CI counterpart to agree with. +# The advisory engine's model and Cyber access program. Pinned here alone: +# the Codex engine has no CI counterpart to agree with. CODEX_MODEL=gpt-6-sol CODEX_PROGRAM=daybreak_blue @@ -150,7 +163,7 @@ run_domain() { return "$status" } -# Each domain's computed verdict, `domain=VERDICT`, for the red team's report; +# Each domain's computed verdict, `domain=VERDICT`, for the advisory report; # a domain that left no fragment is INCONCLUSIVE, as CI computes it. VERDICTS="" run_domain_recorded() { @@ -160,18 +173,18 @@ run_domain_recorded() { return "$status" } -# The command that publishes the red team's report on the commit it audited, -# printed, never run. +# The command that publishes the advisory engine's report on the commit it +# audited, printed, never run. print_codex_report() { local commit="$1" shift echo - echo "==> advisory: a Codex verdict never gates a release (docs/specs/security-audit.md -> \"Red team\")." + echo "==> advisory: a Codex verdict never gates a release (docs/specs/security-audit.md -> \"Advisory engine\")." echo "==> to publish them:" echo "gh workflow run codex-audit-report.yaml -R $GITHUB_REPOSITORY --ref main -f commit=$commit -f model=$CODEX_MODEL $*" } -# The red team's report on the domains just run in the working tree. +# The advisory engine's report on the domains just run in the working tree. print_codex_verdicts() { local commit commit=$(git rev-parse HEAD 2>/dev/null) || commit="" @@ -182,11 +195,19 @@ print_codex_verdicts() { } # The open ledger findings every domain re-verifies, as CI's `List open -# findings` step writes them, read on the operator's own login. Without the -# file each domain computes INCONCLUSIVE, as it does in CI. It reads the -# private tracker and files nothing there. +# findings` step writes them, read on the operator's own login, or copied from +# `AUDIT_OPEN_FINDINGS` with the ledger left unread. Without the file each +# domain computes INCONCLUSIVE, as it does in CI. It reads the private tracker +# and files nothing there. write_open_findings() { local open row + if [ -n "${AUDIT_OPEN_FINDINGS:-}" ]; then + # Through a temporary name, so a variable naming this very file keeps it. + cp "$AUDIT_OPEN_FINDINGS" audit-open-findings.txt.new + mv audit-open-findings.txt.new audit-open-findings.txt + echo "==> open findings: copied from $AUDIT_OPEN_FINDINGS; the ledger is not read" + return 0 + fi rm -f audit-open-findings.txt # `--limit 1000`, not `gh`'s default of 30, which once left an older open # finding unreconciled. @@ -260,14 +281,16 @@ run_canary() { echo "==> keyed by no CI run, so there is no canary run to publish against." return 0 fi + # Without Claude's scorecard the "caught by either" count is left empty, + # which the published recall reads as not computed for that run. if [ -z "$claude_card" ]; then - echo "==> to publish, re-run with --claude-scorecard: the scorecard.json in run $run_key's decrypted canary-transcript." - return 0 + echo "==> no --claude-scorecard (the scorecard.json in run $run_key's decrypted canary-transcript): \"caught by either\" is left empty." fi # shellcheck disable=SC2046,SC2016 print_codex_report "$commit" $(node -e ' const card = JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")); - console.log(`-f canary_run=${process.argv[2]} -f canary_seeded=${card.seeded} -f canary_caught=${card.caught} -f canary_either=${card.either} -f canary_unfinished=${card.unfinished.length}`); + const either = "either" in card ? ` -f canary_either=${card.either}` : ""; + console.log(`-f canary_run=${process.argv[2]} -f canary_seeded=${card.seeded} -f canary_caught=${card.caught}${either} -f canary_unfinished=${card.unfinished.length}`); ' "$stash/scorecard.json" "$run_key") } diff --git a/scripts/security-audit-public-body.mjs b/scripts/security-audit-public-body.mjs index 799ca81fb..0a7f421d6 100644 --- a/scripts/security-audit-public-body.mjs +++ b/scripts/security-audit-public-body.mjs @@ -27,9 +27,10 @@ * Usage: node scripts/security-audit-public-body.mjs --engine codex * --date --run-url --commit --model * --verdicts --canary-run --canary-seeded - * --canary-caught --canary-either --canary-unfinished + * --canary-caught [--canary-either ] --canary-unfinished * --record - * The red team's report (docs/specs/security-audit.md -> "Red team"), from + * The advisory engine's report (docs/specs/security-audit.md -> "Advisory + * engine"), from * `.github/workflows/codex-audit-report.yaml`'s inputs: one comment on the * status issue, and, for a canary, its recall record written to . Every * input is checked against its pattern, the counts against the CI canary run @@ -345,9 +346,9 @@ export function statusBody({ status, fileStatus, date, runUrl, commit, repo, fra /** The outcome an earlier run recorded in the status issue's first line, or null. */ export const previousOutcome = (body) => (typeof body === 'string' ? body.match(STATUS_MARKER)?.[1] ?? null : null); -// --- The red team ------------------------------------------------------------- +// --- The advisory engine ------------------------------------------------------ -/** The marker opening every red-team comment on the status issue. */ +/** The marker opening every advisory-engine comment on the status issue. */ export const CODEX_MARKER = ''; const MODEL = /^[a-z0-9][a-z0-9.-]{0,63}$/; const COUNT = /^(?:0|[1-9]\d{0,5})$/; @@ -366,27 +367,30 @@ export function parseVerdicts(text, root = '.') { } /** - * The red team's canary counts, or null when none were given; throws unless - * all are, each a count, caught no more than either and either no more than - * seeded, and no more unfinished domains than the `domains` a canary runs. + * The advisory engine's canary counts, or null when none were given; throws + * unless all are but `either`, each a count, caught no more than seeded and + * either between them, and no more unfinished domains than the `domains` a + * canary runs. An empty `either` was not computed: null, never zero. */ export function parseCanary({ run = '', seeded = '', caught = '', either = '', unfinished = '' }, domains) { const given = [run, seeded, caught, either, unfinished]; if (given.every((v) => v === '')) return null; valid(run, /^[1-9]\d{0,19}$/, 'canary run'); - const [s, c, e, u] = [seeded, caught, either, unfinished].map((v, i) => Number(valid(v, COUNT, ['canary seeded', 'canary caught', 'canary either', 'canary unfinished'][i]))); - if (c > e || e > s) throw new Error(`invalid canary counts: caught ${c}, either ${e}, seeded ${s}`); + const [s, c, u] = [seeded, caught, unfinished].map((v, i) => Number(valid(v, COUNT, ['canary seeded', 'canary caught', 'canary unfinished'][i]))); + const e = either === '' ? null : Number(valid(either, COUNT, 'canary either')); + if (c > (e ?? s) || (e ?? c) > s) throw new Error(`invalid canary counts: caught ${c}, either ${e ?? 'not computed'}, seeded ${s}`); if (u > domains) throw new Error(`invalid canary counts: ${u} unfinished domains of ${domains}`); return { run, seeded: s, caught: c, either: e, unfinished: u }; } /** - * Checks the red team's canary against the CI canary run it shares seeds with - * — `claudeRun`, the API's view of it, and `claude`, its `canary-recall` - * record, read by `validateRecord` — and returns that record's counts: a run + * Checks the advisory engine's canary against the CI canary run it shares + * seeds with — `claudeRun`, the API's view of it, and `claude`, its + * `canary-recall` record, read by `validateRecord` — and returns that + * record's counts: a run * titled `canaryTitle` of `security-audit.yaml` on `main` at the same commit, - * as many seeds, and an `either` that neither engine's own count exceeds nor - * their sum falls short of. + * as many seeds, and an `either`, when computed, that neither engine's own + * count exceeds nor their sum falls short of. */ export function checkAgainstClaude(canary, { commit, repo, claudeRun, claude, canaryTitle, validateRecord }) { const where = `canary run ${canary.run}`; @@ -395,7 +399,7 @@ export function checkAgainstClaude(canary, { commit, repo, claudeRun, claude, ca if (claudeRun.head_sha !== commit) throw new Error(`${where} seeded ${claudeRun.head_sha}, not ${commit}: the same key seeds the same seeds only over the same commit`); const counts = validateRecord(claude, Number(canary.run)); if (counts.seeded !== canary.seeded) throw new Error(`${where} seeded ${counts.seeded}, not ${canary.seeded}`); - if (canary.either < counts.caught || canary.either > counts.caught + canary.caught) { + if (canary.either !== null && (canary.either < counts.caught || canary.either > counts.caught + canary.caught)) { throw new Error(`either ${canary.either} cannot follow from Claude's ${counts.caught} and Codex's ${canary.caught} of ${canary.seeded}`); } return counts; @@ -403,14 +407,14 @@ export function checkAgainstClaude(canary, { commit, repo, claudeRun, claude, ca const percent = (n, of) => `${Math.round((n / of) * 100)}%`; -/** The comment the red team's report posts on the status issue. */ +/** The comment the advisory engine's report posts on the status issue. */ export function codexComment({ date, runUrl, commit, model, repo, verdicts, canary, claude }) { valid(date, /^\d{4}-\d\d-\d\dT\d\d:\d\dZ$/, 'date'); valid(repo, /^[\w.-]+\/[\w.-]+$/, 'repository'); valid(runUrl, new RegExp(`^https://github\\.com/${repo.replace(/[.]/g, '\\.')}/actions/runs/\\d+$`), 'run URL'); valid(commit, /^[0-9a-f]{40}$/, 'commit'); valid(model, MODEL, 'model'); - const lines = [CODEX_MARKER, '', '## Red team: Codex (advisory)', '', + const lines = [CODEX_MARKER, '', '## Advisory engine: Codex', '', `Reported at ${date} · model \`${model}\` · audited commit [\`${commit.slice(0, 7)}\`](https://github.com/${repo}/commit/${commit}) · [report run](${runUrl})`, '', 'A maintainer ran these prompts on Codex. Its verdicts never gate a release; the nightly Claude audit does.']; if (verdicts.length > 0) { @@ -418,13 +422,14 @@ export function codexComment({ date, runUrl, commit, model, repo, verdicts, cana } if (canary) { const claudeUrl = `https://github.com/${repo}/actions/runs/${canary.run}`; - lines.push('', `**Canary**, on the seeds of [canary run ${canary.run}](${claudeUrl}): Claude caught ${claude.caught} of ${canary.seeded} (${percent(claude.caught, canary.seeded)}), Codex ${canary.caught} (${percent(canary.caught, canary.seeded)}), and the two together ${canary.either} (${percent(canary.either, canary.seeded)}).`); + const together = canary.either === null ? '; how many the two caught together was not computed' : `, and the two together ${canary.either} (${percent(canary.either, canary.seeded)})`; + lines.push('', `**Canary**, on the seeds of [canary run ${canary.run}](${claudeUrl}): Claude caught ${claude.caught} of ${canary.seeded} (${percent(claude.caught, canary.seeded)}), Codex ${canary.caught} (${percent(canary.caught, canary.seeded)})${together}.`); if (canary.unfinished > 0) lines.push('', `${plural(canary.unfinished, 'Codex domain')} left no finished report; ${canary.unfinished === 1 ? 'its' : 'their'} seeds count as missed.`); } return clampIssueBody(`${lines.join('\n')}\n`); } -/** The red team's recall record, which website/scripts/generate-canary-recall.js reads. */ +/** The advisory engine's recall record, which website/scripts/generate-canary-recall.js reads; `either` null when not computed. */ export function codexRecord({ runUrl, commit, model, repo, canary }) { return { claude_run_url: `https://github.com/${repo}/actions/runs/${canary.run}`, commit, model, seeded: canary.seeded, caught: canary.caught, either: canary.either, unfinished: canary.unfinished, run_url: runUrl }; @@ -518,9 +523,9 @@ function publish(args) { } /** - * The red team's report: every input checked before anything is posted. The - * canary's contract is imported here alone, so the nightly's own publishing - * never loads the recall generator. + * The advisory engine's report: every input checked before anything is + * posted. The canary's contract is imported here alone, so the nightly's own + * publishing never loads the recall generator. */ async function publishCodex(args) { const { CANARY_DOMAINS } = await import('./security-audit-canary.mjs'); @@ -536,7 +541,7 @@ async function publishCodex(args) { // An admin's dispatch alone: the bot's `push` can dispatch a workflow too. const actor = valid(process.env.GITHUB_TRIGGERING_ACTOR, /^[A-Za-z0-9-]+(?:\[bot\])?$/, 'triggering actor'); const role = gh(['api', `repos/${repo}/collaborators/${actor}/permission`, '--jq', '.role_name']).trim(); - if (role !== 'admin') throw new Error(`${actor} holds ${role || 'no role'} here; only an admin publishes a red-team report`); + if (role !== 'admin') throw new Error(`${actor} holds ${role || 'no role'} here; only an admin publishes an advisory-engine report`); // Only a commit on this repository's `main`, so a report names code that shipped there. const compare = JSON.parse(gh(['api', `repos/${repo}/compare/${commit}...main`, '--jq', '{status}'])); if (!['identical', 'ahead'].includes(compare.status)) throw new Error(`${commit} is not on main`); @@ -558,7 +563,7 @@ async function publishCodex(args) { if (record) writeFileSync(valid(args.record, /\S/, 'record file'), `${JSON.stringify(record, null, 2)}\n`); const dir = mkdtempSync(join(process.env.RUNNER_TEMP || tmpdir(), 'codex-public-')); gh(['issue', 'comment', String(statuses[0].number), ...R, '--body-file', bodyFile(dir, 'codex.md', body)]); - console.log(`Status issue #${statuses[0].number}: red-team report posted${record ? ', recall record written' : ''}.`); + console.log(`Status issue #${statuses[0].number}: advisory-engine report posted${record ? ', recall record written' : ''}.`); } async function main(argv) { diff --git a/scripts/security-audit.test.mjs b/scripts/security-audit.test.mjs index f3d871472..f1a1da412 100644 --- a/scripts/security-audit.test.mjs +++ b/scripts/security-audit.test.mjs @@ -1522,7 +1522,7 @@ for (const [arg, runsState, runsDomain] of [['github-state', true, false], ['ci- }); } -// The red team: the same prompts on Codex, each domain's computed verdict +// The advisory engine: the same prompts on Codex, each domain's computed verdict // gathered into the publishing command, which is printed and never run. for (const [args, verdicts, expected] of [[['remote'], 'remote=PASS', 0], [[], 'supply-chain=PASS ci-and-secrets=PASS remote=PASS local=INCONCLUSIVE hosted=PASS', 1]]) { test(`local runner, Codex engine: ${args[0] ?? 'every domain'}`, (t) => { @@ -1544,6 +1544,35 @@ for (const [args, verdicts, expected] of [[['remote'], 'remote=PASS', 0], [[], ' }); } +// A list written on another login: the domains read it, and the ledger is +// never listed, so this run's `gh` login needs no read on it. +test('local runner: AUDIT_OPEN_FINDINGS hands the domains a file, and the ledger is not read', (t) => { + const { dir, env } = fixture(t); + localRunnerFixture(dir); + stubGh(dir); + const title = '[audit-finding 0123456789ab] FAIL S9: from the file'; + mkdirSync(join(dir, 'elsewhere')); + writeFileSync(join(dir, 'elsewhere/open.txt'), `${title}\n`); + stub(dir, 'codex', ` + const fs = require('node:fs'); + const argv = process.argv.slice(2); + const output = argv[argv.length - 1].match(/\\*\\*Output file:\\*\\* \\x60([^\\x60]+)\\x60/)[1]; + fs.writeFileSync('seen-open-findings', fs.readFileSync('audit-open-findings.txt')); + fs.writeFileSync(output, ${JSON.stringify(Object.fromEntries(fragments.map((f) => [f, fragmentText(f)])))}[output]); + `); + // A relative path is the operator's, from where they ran the script. + const result = spawnSync('bash', ['../scripts/security-audit-local.sh', '--engine', 'codex', 'remote'], + { cwd: join(dir, 'elsewhere'), env: { ...env, AUDIT_OPEN_FINDINGS: 'open.txt', GH_LEDGER_OPEN: '7 [audit-finding ba9876543210] FAIL S5: from the ledger' }, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(join(dir, 'seen-open-findings'), 'utf8'), `${title}\n`); + assert.equal(existsSync(join(dir, 'gh-calls.jsonl')) ? readFileSync(join(dir, 'gh-calls.jsonl'), 'utf8') : '', ''); + + const missing = spawnSync('bash', ['scripts/security-audit-local.sh', '--engine', 'codex', 'remote'], + { cwd: dir, env: { ...env, AUDIT_OPEN_FINDINGS: 'nowhere.txt' }, encoding: 'utf8' }); + assert.equal(missing.status, 64); + assert.match(missing.stderr, /AUDIT_OPEN_FINDINGS names no file: nowhere\.txt/); +}); + test('local runner: an unknown engine is refused before anything runs', (t) => { const { dir, env } = fixture(t); localRunnerFixture(dir); @@ -1812,11 +1841,11 @@ test('the model split holds in CI and locally, over the same prompt files', () = assert.match(local, /cat "\$AUDIT_DIR\/_preamble\.md"; echo; cat "\$AUDIT_DIR\/\$domain\.md"/); }); -// "Red team": the Codex engine is pinned and sandboxed, and its report +// "Advisory engine": the Codex engine is pinned and sandboxed, and its report // workflow publishes only what its builder checked, for an admin, from main. const codexWorkflow = readFileSync(join(repo, '.github/workflows/codex-audit-report.yaml'), 'utf8'); -test('the red team runs sandboxed on its pinned model and program, and gates nothing', () => { +test('the advisory engine runs sandboxed on its pinned model and program, and gates nothing', () => { assert.match(local, /^CODEX_MODEL=gpt-6-sol$/m); assert.match(local, /^CODEX_PROGRAM=daybreak_blue$/m); const exec = local.match(/codex exec [^\n]*\\\n[^\n]*/)?.[0] ?? ''; @@ -1839,12 +1868,12 @@ test('the red team runs sandboxed on its pinned model and program, and gates not assert.equal(spawnSync('bash', ['-c', steps[0].run], { env: { ...process.env, GITHUB_REF: ref } }).status, status, ref); } const posting = steps.filter((step) => /\bgh\b|security-audit-public-body/.test(step.run ?? '')); - assert.deepEqual(posting.map((step) => step.name), ['Publish the red-team report']); + assert.deepEqual(posting.map((step) => step.name), ['Publish the advisory-engine report']); assert.match(posting[0].run, /node scripts\/security-audit-public-body\.mjs --engine codex /); assert.equal(doc.jobs.publish.steps.find((step) => step.uses?.startsWith('actions/checkout@')).with['persist-credentials'], false); }); -/** The builder's red-team path in a fixture tree, over a `gh` stub answering what it reads. */ +/** The builder's advisory-engine path in a fixture tree, over a `gh` stub answering what it reads. */ function codexFixture(t, { role = 'admin', compare = 'ahead', run = {}, record = {} } = {}) { const { dir, env } = fixture(t); copyFileSync(join(repo, 'scripts/security-audit-canary.mjs'), join(dir, 'scripts/security-audit-canary.mjs')); @@ -1875,7 +1904,7 @@ const CODEX_ARGS = ['--engine', 'codex', '--date', '2026-10-20T12:00Z', '--run-u const runCodex = (dir, env, args = CODEX_ARGS) => spawnSync(process.execPath, ['scripts/security-audit-public-body.mjs', ...args], { cwd: dir, env, encoding: 'utf8' }); const withArg = (name, value) => CODEX_ARGS.map((a, i) => (CODEX_ARGS[i - 1] === name ? value : a)); -test('the red-team report comments once on the status issue and writes its recall record', (t) => { +test('the advisory-engine report comments once on the status issue and writes its recall record', (t) => { const { dir, env } = codexFixture(t); const result = runCodex(dir, env); assert.equal(result.status, 0, result.stderr); @@ -1890,6 +1919,18 @@ test('the red-team report comments once on the status issue and writes its recal seeded: 8, caught: 5, either: 7, unfinished: 0, run_url: 'https://github.com/fixture/repo/actions/runs/5' }); }); +// Published without Claude's scorecard: the either count is not computed, +// which the record keeps as null and the comment says, never as zero. +test('the advisory-engine report publishes a canary whose either count was not computed', (t) => { + const { dir, env } = codexFixture(t); + const result = runCodex(dir, env, withArg('--canary-either', '')); + assert.equal(result.status, 0, result.stderr); + const comment = readFileSync(join(dir, 'gh-calls.jsonl'), 'utf8').trim().split('\n').map((l) => JSON.parse(l)) + .find(({ args }) => args[0] === 'issue' && args[1] === 'comment'); + assert.match(comment.body, /Claude caught 6 of 8 \(75%\), Codex 5 \(63%\); how many the two caught together was not computed\./); + assert.equal(JSON.parse(readFileSync(join(dir, 'codex-recall.json'), 'utf8')).either, null); +}); + for (const [name, options, args, message] of [ ['a dispatch by a non-admin', { role: 'write' }, CODEX_ARGS, /only an admin/], ['a commit off main', { compare: 'diverged' }, CODEX_ARGS, /is not on main/], @@ -1903,8 +1944,9 @@ for (const [name, options, args, message] of [ ['a model name carrying text', {}, withArg('--model', 'gpt-6-sol see src/x.ts:12'), /invalid model/], ['caught above either', {}, withArg('--canary-caught', '8'), /invalid canary counts/], ['half the canary counts', {}, withArg('--canary-unfinished', ''), /invalid canary unfinished/], + ['caught above seeded, either not computed', {}, withArg('--canary-either', '').map((a, i, all) => (all[i - 1] === '--canary-caught' ? '9' : a)), /invalid canary counts/], ]) { - test(`the red-team report posts nothing for ${name}`, (t) => { + test(`the advisory-engine report posts nothing for ${name}`, (t) => { const { dir, env } = codexFixture(t, options); const result = runCodex(dir, env, args); assert.notEqual(result.status, 0); diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index 9491bc82e..bf7a651e7 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -24,7 +24,7 @@ "docs/specs/remote-network.md": 2400, "docs/specs/remote-security-model.md": 5700, "docs/specs/reopen.md": 1200, - "docs/specs/security-audit.md": 5150, + "docs/specs/security-audit.md": 5250, "docs/specs/security-ci.md": 3400, "docs/specs/security-hosted.md": 3050, "docs/specs/security-local.md": 6100, diff --git a/scripts/workflow-lint.test.mjs b/scripts/workflow-lint.test.mjs index 19e2bd854..0e1cc81a7 100644 --- a/scripts/workflow-lint.test.mjs +++ b/scripts/workflow-lint.test.mjs @@ -36,8 +36,8 @@ const cases = [ ['a non-agent workflow with write-all', edit('ci.yml', 'permissions:\n contents: read', 'permissions: write-all'), 'holds write on *'], ['release.yml build job writes packages', edit('release.yml', ' id-token: write\n attestations: write\n strategy:', ' id-token: write\n attestations: write\n packages: write\n strategy:'), 'build-standalone` holds write on packages'], ['actions: write beyond the audit dispatch', edit('release.yml', ' id-token: write\n attestations: write\n steps:', ' id-token: write\n attestations: write\n actions: write\n steps:'), 'build-vscode` holds write on actions'], - ['the red-team report writes contents', edit('codex-audit-report.yaml', ' contents: read\n issues: write', ' contents: write\n issues: write'), 'codex-audit-report.yaml job `publish` holds write on contents'], - ['issues: write on another job of the red-team report', edit('codex-audit-report.yaml', ' publish:\n', ' other:\n'), 'codex-audit-report.yaml job `other` holds write on issues'], + ['the advisory-engine report writes contents', edit('codex-audit-report.yaml', ' contents: read\n issues: write', ' contents: write\n issues: write'), 'codex-audit-report.yaml job `publish` holds write on contents'], + ['issues: write on another job of the advisory-engine report', edit('codex-audit-report.yaml', ' publish:\n', ' other:\n'), 'codex-audit-report.yaml job `other` holds write on issues'], ['an agent-managed job with actions: write', edit('tend-review.yaml', ' actions: read\n', ' actions: write\n'), 'tend-review.yaml job `review` holds write on actions'], ['an agent-managed job with checks: write', edit('security-audit.yaml', ' actions: read\n issues: write', ' actions: read\n checks: write\n issues: write'), 'security-audit.yaml job `audit` holds write on checks'], ['an agent-managed job with id-token: write', edit('security-audit.yaml', ' actions: read\n issues: write', ' actions: read\n issues: write\n id-token: write'), 'security-audit.yaml job `audit` holds write on id-token'], diff --git a/website/scripts/generate-canary-recall.js b/website/scripts/generate-canary-recall.js index 483966c11..97b8b9f56 100644 --- a/website/scripts/generate-canary-recall.js +++ b/website/scripts/generate-canary-recall.js @@ -8,11 +8,11 @@ * runs on `main` created in the last `WINDOW_DAYS` days, downloads each one's * `canary-recall` artifact — never `canary-transcript`, which is ciphertext and * not ours to open here — validates it, and writes - * website/src/data/canary-recall.json. The red team's records ride along: - * each successful `codex-audit-report.yaml` run's `codex-recall`, paired - * with the canary run whose seeds it shares (docs/specs/security-audit.md -> - * "Red team"). Run by hand with the dependency - * snapshots (docs/specs/deploy.md -> "Release checklist"), because + * website/src/data/canary-recall.json. The advisory engine's records ride + * along: each successful `codex-audit-report.yaml` run's `codex-recall`, + * paired with the canary run whose seeds it shares + * (docs/specs/security-audit.md -> "Advisory engine"). Run by hand with the + * dependency snapshots (docs/specs/deploy.md -> "Release checklist"), because * downloading an artifact needs a token even on a public repository, and the * site is built by Cloudflare Pages, which holds none. */ @@ -65,10 +65,11 @@ export function validateRecord(record, id) { } /** - * One red-team run's `codex-recall.json`, checked as `validateRecord` checks a - * canary's: exactly the fields the builder writes, counts with caught no more - * than either and either no more than seeded, and links to this very run and - * to a run on this repository. Returns the counts and the paired run's URL. + * One advisory-engine run's `codex-recall.json`, checked as `validateRecord` + * checks a canary's: exactly the fields the builder writes, counts with + * caught no more than seeded and either between them, and links to this very + * run and to a run on this repository. A null `either` was not computed and + * stays null, never zero. Returns the counts and the paired run's URL. */ export function validateCodexRecord(record, id) { const where = `run ${id}'s ${CODEX_ARTIFACT}`; @@ -76,9 +77,11 @@ export function validateCodexRecord(record, id) { const keys = Object.keys(record).sort(); if (keys.join() !== CODEX_FIELDS.join()) throw new Error(`${where} has fields ${keys.join(', ')}, not ${CODEX_FIELDS.join(', ')}`); for (const field of ['seeded', 'caught', 'either', 'unfinished']) { - if (!isCount(record[field])) throw new Error(`${where}: ${field} is not a non-negative integer`); + if (!isCount(record[field]) && !(field === 'either' && record.either === null)) throw new Error(`${where}: ${field} is not a non-negative integer`); + } + if (record.caught > (record.either ?? record.seeded) || (record.either ?? record.caught) > record.seeded) { + throw new Error(`${where}: caught ${record.caught}, either ${record.either ?? 'not computed'}, seeded ${record.seeded}`); } - if (record.caught > record.either || record.either > record.seeded) throw new Error(`${where}: caught ${record.caught}, either ${record.either}, seeded ${record.seeded}`); if (record.unfinished > CANARY_DOMAINS.length) throw new Error(`${where}: ${record.unfinished} unfinished domains of ${CANARY_DOMAINS.length}`); if (record.run_url !== runUrl(id)) throw new Error(`${where} links ${JSON.stringify(record.run_url)}, not ${runUrl(id)}`); const paired = /^https:\/\/github\.com\/diffplug\/dormouse\/actions\/runs\/([1-9]\d*)$/.exec(record.claude_run_url); @@ -87,10 +90,10 @@ export function validateCodexRecord(record, id) { } /** - * Each recorded canary run's red-team counts, by the canary run's URL: the - * newest red-team run paired with it. Throws on a pairing that cannot follow - * from the canary's own record — other seeds, or an `either` below Claude's - * count or above the two engines' sum. + * Each recorded canary run's advisory-engine counts, by the canary run's URL: + * the newest advisory-engine run paired with it. Throws on a pairing that + * cannot follow from the canary's own record — other seeds, or a computed + * `either` below Claude's count or above the two engines' sum. */ function pairCodex(recorded, codex) { const byUrl = new Map(); @@ -98,8 +101,9 @@ function pairCodex(recorded, codex) { const counts = validateCodexRecord(c.record, c.id); const claude = recorded.find((r) => r.url === counts.claudeUrl); if (!claude) continue; - if (counts.seeded !== claude.seeded || counts.either < claude.caught || counts.either > claude.caught + counts.caught) { - throw new Error(`run ${c.id}'s ${CODEX_ARTIFACT} cannot pair with ${claude.url}: seeded ${counts.seeded} of ${claude.seeded}, either ${counts.either} beside Claude's ${claude.caught}`); + if (counts.seeded !== claude.seeded + || (counts.either !== null && (counts.either < claude.caught || counts.either > claude.caught + counts.caught))) { + throw new Error(`run ${c.id}'s ${CODEX_ARTIFACT} cannot pair with ${claude.url}: seeded ${counts.seeded} of ${claude.seeded}, either ${counts.either ?? 'not computed'} beside Claude's ${claude.caught}`); } const { claudeUrl: _claudeUrl, seeded: _seeded, ...kept } = counts; byUrl.set(claude.url, kept); @@ -110,7 +114,7 @@ function pairCodex(recorded, codex) { /** * The snapshot, from every canary run listed: `{ id, createdAt, record }`, * `record` null for a run that left no `canary-recall` artifact; and every - * red-team run listed, `{ id, record }`, each record a `codex-recall`. + * advisory-engine run listed, `{ id, record }`, each record a `codex-recall`. * * Every run in the window counts as its record has it. A domain that left no * finished report caught only what its partial report cites, so the seeds @@ -156,8 +160,9 @@ function listCanaryRuns(since) { } /** - * The successful red-team runs on `main` created since `since`. Success alone - * is an admin's dispatch: `codex-audit-report.yaml` fails any other. + * The successful advisory-engine runs on `main` created since `since`. + * Success alone is an admin's dispatch: `codex-audit-report.yaml` fails any + * other. */ function listCodexRuns(since) { const query = `repos/${REPO}/actions/workflows/${CODEX_WORKFLOW}/runs?branch=main&status=success&created=%3E%3D${since}&per_page=100`; @@ -190,7 +195,7 @@ function main() { writeFileSync(out, `${JSON.stringify(snapshot, null, 2)}\n`); console.log(`canary recall: ${snapshot.caught}/${snapshot.seeded} over ${snapshot.runs.length} run(s) since ${since}` + `${snapshot.unrecorded.length ? `; ${snapshot.unrecorded.length} run(s) left no record` : ''}` + - `; ${snapshot.runs.filter((r) => r.codex).length} paired with the red team -> ${out}`); + `; ${snapshot.runs.filter((r) => r.codex).length} paired with the advisory engine -> ${out}`); } if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/website/scripts/generate-canary-recall.test.js b/website/scripts/generate-canary-recall.test.js index 1f142f405..26fb8038f 100644 --- a/website/scripts/generate-canary-recall.test.js +++ b/website/scripts/generate-canary-recall.test.js @@ -72,13 +72,20 @@ const codex = (id, paired, fields = {}) => ({ claude_run_url: URL(paired), commi seeded: 8, caught: 5, either: 7, unfinished: 0, run_url: URL(id), ...fields }); describe('validateCodexRecord', () => { - it('reads exactly the fields the red-team builder writes', () => { + it('reads exactly the fields the advisory-engine builder writes', () => { const written = codexRecord({ runUrl: URL(9), commit: 'a'.repeat(40), model: 'gpt-6-sol', repo: 'diffplug/dormouse', canary: { run: '7', seeded: 8, caught: 5, either: 7, unfinished: 0 } }); expect(Object.keys(written).sort()).toEqual(CODEX_FIELDS); expect(validateCodexRecord(written, 9)).toEqual({ url: URL(9), claudeUrl: URL(7), seeded: 8, caught: 5, either: 7, unfinished: 0 }); }); + it('keeps an either count that was not computed as null, never zero', () => { + const written = codexRecord({ runUrl: URL(9), commit: 'a'.repeat(40), model: 'gpt-6-sol', repo: 'diffplug/dormouse', + canary: { run: '7', seeded: 8, caught: 5, either: null, unfinished: 0 } }); + expect(validateCodexRecord(written, 9)).toMatchObject({ caught: 5, either: null }); + expect(() => validateCodexRecord(codex(9, 7, { either: undefined }), 9)).toThrow(/either is not a non-negative integer/); + }); + it('rejects a record out of shape or linked elsewhere', () => { for (const [bad, message] of [ [{ ...codex(9, 7), note: 'x' }, /has fields/], @@ -91,8 +98,8 @@ describe('validateCodexRecord', () => { }); }); -describe('summarize with the red team', () => { - it('pairs each canary run with its newest red-team record, and ignores one paired outside the window', () => { +describe('summarize with the advisory engine', () => { + it('pairs each canary run with its newest advisory-engine record, and ignores one paired outside the window', () => { const snapshot = summarize([{ id: 7, createdAt: daysAgo(3), record: record(7) }], { now: NOW, codex: [ { id: 9, record: codex(9, 7, { caught: 2, either: 6 }) }, { id: 11, record: codex(11, 7) }, @@ -102,6 +109,13 @@ describe('summarize with the red team', () => { codex: { url: URL(11), caught: 5, either: 7, unfinished: 0 } }]); }); + it('pairs a record whose either count was not computed, and keeps it so', () => { + const snapshot = summarize([{ id: 7, createdAt: daysAgo(3), record: record(7) }], { now: NOW, codex: [ + { id: 9, record: codex(9, 7, { caught: 2, either: null }) }, + ] }); + expect(snapshot.runs[0].codex).toEqual({ url: URL(9), caught: 2, either: null, unfinished: 0 }); + }); + it('refuses a pairing its canary record cannot support', () => { const run = [{ id: 7, createdAt: daysAgo(3), record: record(7) }]; for (const fields of [{ seeded: 6, either: 6 }, { either: 5 }, { caught: 0, either: 7 }]) { @@ -110,7 +124,7 @@ describe('summarize with the red team', () => { }); }); -it('selects red-team runs by the title their workflow gives them', () => { +it('selects advisory-engine runs by the title their workflow gives them', () => { const workflow = readFileSync(resolve(import.meta.dirname, '../../.github/workflows/codex-audit-report.yaml'), 'utf8'); expect(workflow).toMatch(new RegExp(`^run-name: ${CODEX_TITLE.replace(/[()]/g, '\\$&')}$`, 'm')); }); diff --git a/website/src/components/CanaryRecall.test.tsx b/website/src/components/CanaryRecall.test.tsx index 1b2d7117a..066326be7 100644 --- a/website/src/components/CanaryRecall.test.tsx +++ b/website/src/components/CanaryRecall.test.tsx @@ -38,9 +38,9 @@ const SNAPSHOT: CanaryRecallSnapshot = { { at: "2026-09-30T00:00:00Z", url: RUN(6) }, ], }; -// The red team paired with runs 2 and 3: Oct 5's week is all paired, Oct 12's -// in part, so its either line adds run 5's Claude catches alone. -const RED_TEAM: CanaryRecallSnapshot = { +// The advisory engine paired with runs 2 and 3: Oct 5's week is all paired, +// Oct 12's in part, so its either line adds run 5's Claude catches alone. +const ADVISORY: CanaryRecallSnapshot = { ...SNAPSHOT, runs: SNAPSHOT.runs.map((run) => run.url === RUN(2) @@ -107,26 +107,26 @@ describe("recallWeeks", () => { }); }); -describe("the red team", () => { +describe("the advisory engine", () => { it("scores Codex over its paired runs, and either over every counted run", () => { - const week = Object.fromEntries(recallWeeks(RED_TEAM).map((w) => [w.start, w])); + const week = Object.fromEntries(recallWeeks(ADVISORY).map((w) => [w.start, w])); expect(week["2026-10-05"].codex).toEqual({ runs: 1, seeded: 8, caught: 5, percent: 63, either: 8, eitherPercent: 100 }); - // Run 5 had no red team: its 8 Claude catches join run 3's 7 caught by either. + // Codex did not see run 5: its 8 Claude catches join run 3's 7 caught by either. expect(week["2026-10-12"].codex).toEqual({ runs: 1, seeded: 8, caught: 4, percent: 50, either: 15, eitherPercent: 94 }); expect(week["2026-09-21"].codex).toBeNull(); - for (const w of recallWeeks(RED_TEAM)) if (w.codex) expect(w.codex.eitherPercent).toBeGreaterThanOrEqual(w.percent!); + for (const w of recallWeeks(ADVISORY)) if (w.codex) expect(w.codex.eitherPercent).toBeGreaterThanOrEqual(w.percent!); }); it("says all three in each description and run", () => { - const week = Object.fromEntries(recallWeeks(RED_TEAM).map((w) => [w.start, w])); + const week = Object.fromEntries(recallWeeks(ADVISORY).map((w) => [w.start, w])); expect(weekDescription(week["2026-10-05"])).toBe( "Week of Oct 5: caught 7 of 8 over 1 run (88%); Codex caught 5 of 8 over 1 run (63%); caught by either: 8 of 8 (100%)", ); expect(week["2026-10-12"].runs.map(runLabel)).toEqual(["Oct 12: 6 of 8*; Codex 4*, either 7", "Oct 14: no record", "Oct 16: 8 of 8"]); }); - it("draws three lines under a legend only once the red team has a point", () => { - const html = renderToStaticMarkup(); + it("draws three lines under a legend only once the advisory engine has a point", () => { + const html = renderToStaticMarkup(); const shown = text(html); for (const label of ["Claude, the release gate", "Codex, advisory", "Caught by either"]) expect(shown).toContain(label); expect(shown).toContain("its line covers only those runs"); @@ -144,6 +144,31 @@ describe("the red team", () => { }); }); +describe("an either count that was not computed", () => { + // Run 3's Codex canary was published without Claude's scorecard. + const UNCOMPUTED: CanaryRecallSnapshot = { + ...ADVISORY, + runs: ADVISORY.runs.map((run) => (run.url === RUN(3) ? { ...run, codex: { ...run.codex!, either: null } } : run)), + }; + const week = Object.fromEntries(recallWeeks(UNCOMPUTED).map((w) => [w.start, w])); + + it("leaves that week's either empty, never zero, and keeps Codex's own count", () => { + expect(week["2026-10-12"].codex).toEqual({ runs: 1, seeded: 8, caught: 4, percent: 50, either: null, eitherPercent: null }); + expect(week["2026-10-05"].codex).toMatchObject({ either: 8, eitherPercent: 100 }); + expect(weekDescription(week["2026-10-12"])).toContain("caught by either: not computed"); + expect(week["2026-10-12"].runs[0] && runLabel(week["2026-10-12"].runs[0])).toBe("Oct 12: 6 of 8*; Codex 4*, either not computed"); + }); + + it("plots no point on the either line that week", () => { + const html = renderToStaticMarkup(); + const lines = [...html.matchAll(/ points); + expect(lines).toEqual([`${slot(2, 4)},0`, `${slot(2, 4)},37 ${slot(3, 4)},50`, `${slot(0, 4)},50`, `${slot(2, 4)},12 ${slot(3, 4)},12`]); + // One diamond in the legend, one for Oct 5's either point; none for Oct 12's. + expect(html.match(/rotate\(45 /g)).toHaveLength(2); + expect(renderToStaticMarkup().match(/rotate\(45 /g)).toHaveLength(3); + }); +}); + describe("week labels", () => { it("say counts in words, never with a slash, and list each run", () => { const week = Object.fromEntries(recallWeeks(SNAPSHOT).map((w) => [w.start, w])); diff --git a/website/src/components/CanaryRecall.tsx b/website/src/components/CanaryRecall.tsx index e90c889fc..a6b77b45e 100644 --- a/website/src/components/CanaryRecall.tsx +++ b/website/src/components/CanaryRecall.tsx @@ -11,8 +11,11 @@ export type CanaryRecallSnapshot = { unrecorded: { at: string; url: string }[]; }; -/** The red team's counts over one canary run's seeds (docs/specs/security-audit.md -> "Red team"). */ -export type CodexCounts = { url: string; caught: number; either: number; unfinished: number }; +/** + * The advisory engine's counts over one canary run's seeds + * (docs/specs/security-audit.md -> "Advisory engine"); `either` null when not computed. + */ +export type CodexCounts = { url: string; caught: number; either: number | null; unfinished: number }; /** One run inside a week. `seeded` and `caught` are null for a run that left no record. */ export type RecallRun = { @@ -41,11 +44,19 @@ export type RecallWeek = { /** Runs that week that left no record, and so add nothing to its counts. */ unrecorded: number; /** - * The red team, over that week's runs it was paired with; null when none - * was. `either` is over every counted run, a run Codex did not see adding - * Claude's catches alone, so it never falls below `percent`. + * The advisory engine, over that week's runs it was paired with; null when + * none was. `either` is over every counted run, a run Codex did not see + * adding Claude's catches alone, so it never falls below `percent`; null + * when a paired run did not compute it, which plots no point that week. */ - codex: { runs: number; seeded: number; caught: number; percent: number; either: number; eitherPercent: number } | null; + codex: { + runs: number; + seeded: number; + caught: number; + percent: number; + either: number | null; + eitherPercent: number | null; + } | null; }; const DAY = 86_400_000; @@ -87,7 +98,9 @@ export function recallWeeks(snapshot: CanaryRecallSnapshot): RecallWeek[] { const paired = counted.filter((run) => run.codex); const codexSeeded = paired.reduce((n, run) => n + run.seeded!, 0); const codexCaught = paired.reduce((n, run) => n + run.codex!.caught, 0); - const either = counted.reduce((n, run) => n + (run.codex ? run.codex.either : run.caught!), 0); + const either = paired.some((run) => run.codex!.either === null) + ? null + : counted.reduce((n, run) => n + (run.codex ? run.codex.either! : run.caught!), 0); weeks.push({ start: isoDay(monday), runs: mine, @@ -103,7 +116,7 @@ export function recallWeeks(snapshot: CanaryRecallSnapshot): RecallWeek[] { caught: codexCaught, percent: Math.round((codexCaught / codexSeeded) * 100), either, - eitherPercent: Math.round((either / seeded) * 100), + eitherPercent: either === null ? null : Math.round((either / seeded) * 100), } : null, }); @@ -128,7 +141,9 @@ export function weekDescription(week: RecallWeek): string { if (week.codex) { parts.push( `Codex caught ${week.codex.caught} of ${week.codex.seeded} over ${plural(week.codex.runs, "run")} (${week.codex.percent}%)`, - `caught by either: ${week.codex.either} of ${week.seeded} (${week.codex.eitherPercent}%)`, + week.codex.either === null + ? "caught by either: not computed" + : `caught by either: ${week.codex.either} of ${week.seeded} (${week.codex.eitherPercent}%)`, ); } if (week.unfinished > 0) parts.push(`${plural(week.unfinished, "run")} with a domain that left no finished report`); @@ -139,7 +154,9 @@ export function weekDescription(week: RecallWeek): string { /** One run in a week's list, in words. */ export function runLabel(run: RecallRun): string { const counts = run.seeded === null ? "no record" : `${run.caught} of ${run.seeded}${run.unfinished > 0 ? "*" : ""}`; - const codex = run.codex ? `; Codex ${run.codex.caught}${run.codex.unfinished > 0 ? "*" : ""}, either ${run.codex.either}` : ""; + const codex = run.codex + ? `; Codex ${run.codex.caught}${run.codex.unfinished > 0 ? "*" : ""}, either ${run.codex.either ?? "not computed"}` + : ""; return `${dayOf(run.at)}: ${counts}${codex}`; } @@ -225,7 +242,7 @@ function SeriesMarker({ series, cx, cy, r }: { series: Series; cx: number; cy: n return ; } -/** The legend, shown once any week has a red-team point: identity is never color alone. */ +/** The legend, shown once any week has an advisory-engine point: identity is never color alone. */ function RecallLegend() { return (
    @@ -288,20 +305,24 @@ function RecallGraph({ weeks }: { weeks: RecallWeek[] }) { )), )} - {/* The red team's points: the week's own point below opens its runs and describes all three. */} + {/* The advisory engine's points: the week's own point below opens its runs and describes all three. */} {weeks.flatMap((week, i) => week.codex - ? (["codex", "either"] as const).map((series) => ( - - )) + ? ([["codex", week.codex.percent], ["either", week.codex.eitherPercent]] as const).flatMap(([series, value]) => + value === null + ? [] + : [ + , + ], + ) : [], )}
      @@ -382,7 +403,7 @@ export default function CanaryRecall({ snapshot }: { snapshot: CanaryRecallSnaps const weeks = recallWeeks(snapshot); const any = snapshot.runs.length + snapshot.unrecorded.length > 0; const unfinished = snapshot.runs.some((run) => run.unfinished > 0 || (run.codex?.unfinished ?? 0) > 0); - const redTeam = weeks.some((week) => week.codex !== null); + const advisory = weeks.some((week) => week.codex !== null); return (