From aa66cbb10c99162dbabddbae8d9303d03c4980b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 00:44:16 +0000 Subject: [PATCH 1/3] feat(bench): add a feature-building benchmark for Proof's context cost benchmarks/proof-build asks whether Proof in context helps or hurts an agent that plans or builds a feature from a product-sense request. - A deterministic generator builds Tessera, a zero-dependency TypeScript design-editor core (79 source files, 132 tests), with 84 commits by five authors and a 7-Effort Proof graph: rules recorded only in Proof, rules also visible in code, five drifts the code moved past without Proof, two proper supersessions, and a 10x filler variant. A none variant has the same code with no Proof folder and no trailers. - 18 build tasks (6 proof-only, 5 code-visible, 4 stale, 3 control) and 8 plan tasks with rubrics, phrased as product requests that name no file, symbol or record. - Arms: none, skill, brief, find, findcode, dump and dump10x, identical apart from Proof access. - grade.mjs applies each session's diff to a clean tree and runs hidden acceptance, constraint and regression tests with no network; judge.mjs scores blinded plans; compare.mjs pairs arms against none with sign tests and Holm correction. Hidden tests and reference and naive patches stay outside the repository until the runs are done. The offline gate passes: each base fails acceptance, each reference passes, and each naive patch fails what its category predicts (54 patches, 0 failures). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DVVnhYL1Bz7mCfEKDoyLMW --- benchmarks/proof-build/DESIGN.md | 598 ++++ benchmarks/proof-build/README.md | 170 ++ benchmarks/proof-build/scripts/arms.mjs | 108 + benchmarks/proof-build/scripts/check-bed.mjs | 263 ++ benchmarks/proof-build/scripts/common.mjs | 574 ++++ benchmarks/proof-build/scripts/compare.mjs | 459 ++++ benchmarks/proof-build/scripts/gate.mjs | 329 +++ benchmarks/proof-build/scripts/gen-bed.mjs | 832 ++++++ benchmarks/proof-build/scripts/grade.mjs | 463 ++++ benchmarks/proof-build/scripts/inject.mjs | 171 ++ benchmarks/proof-build/scripts/judge.mjs | 477 ++++ benchmarks/proof-build/scripts/run.mjs | 510 ++++ benchmarks/proof-build/spec/app/.gitignore | 3 + benchmarks/proof-build/spec/app/AGENTS.md | 10 + benchmarks/proof-build/spec/app/README.md | 39 + .../proof-build/spec/app/docs/plugins.md | 47 + .../spec/app/src/canvas/gestures.ts | 163 ++ .../spec/app/src/canvas/handles.ts | 45 + .../spec/app/src/canvas/viewport.ts | 55 + .../spec/app/src/commands/batch.ts | 10 + .../spec/app/src/commands/builtin/align.ts | 130 + .../spec/app/src/commands/builtin/delete.ts | 79 + .../spec/app/src/commands/builtin/group.ts | 52 + .../spec/app/src/commands/builtin/lock.ts | 25 + .../spec/app/src/commands/builtin/move.ts | 33 + .../spec/app/src/commands/builtin/nudge.ts | 37 + .../spec/app/src/commands/builtin/rename.ts | 14 + .../spec/app/src/commands/builtin/reorder.ts | 57 + .../spec/app/src/commands/builtin/resize.ts | 60 + .../spec/app/src/commands/builtin/set-prop.ts | 25 + .../app/src/commands/builtin/visibility.ts | 28 + .../proof-build/spec/app/src/commands/bus.ts | 133 + .../spec/app/src/commands/coalesce.ts | 41 + .../spec/app/src/commands/history.ts | 108 + .../spec/app/src/commands/palette.ts | 211 ++ .../spec/app/src/commands/registry.ts | 61 + .../spec/app/src/commands/types.ts | 52 + benchmarks/proof-build/spec/app/src/editor.ts | 193 ++ .../spec/app/src/keys/bindings/canvas.ts | 51 + .../spec/app/src/keys/bindings/layers.ts | 35 + .../spec/app/src/keys/bindings/panel.ts | 5 + .../proof-build/spec/app/src/keys/chords.ts | 76 + .../spec/app/src/keys/dispatcher.ts | 118 + .../proof-build/spec/app/src/keys/keymap.ts | 60 + .../proof-build/spec/app/src/keys/scopes.ts | 31 + .../proof-build/spec/app/src/layers/drag.ts | 37 + .../proof-build/spec/app/src/layers/layers.ts | 104 + .../proof-build/spec/app/src/layers/rename.ts | 17 + .../spec/app/src/layers/tree-view.ts | 54 + .../spec/app/src/model/document.ts | 183 ++ .../proof-build/spec/app/src/model/factory.ts | 160 ++ .../spec/app/src/model/geometry.ts | 122 + .../proof-build/spec/app/src/model/query.ts | 53 + .../proof-build/spec/app/src/model/style.ts | 61 + .../proof-build/spec/app/src/model/text.ts | 59 + .../spec/app/src/model/transform.ts | 78 + .../proof-build/spec/app/src/model/tree.ts | 103 + .../proof-build/spec/app/src/model/types.ts | 113 + .../proof-build/spec/app/src/panel/commit.ts | 38 + .../proof-build/spec/app/src/panel/fields.ts | 56 + .../proof-build/spec/app/src/panel/mixed.ts | 27 + .../spec/app/src/panel/properties.ts | 128 + .../proof-build/spec/app/src/panel/schema.ts | 45 + .../spec/app/src/panel/sections/appearance.ts | 26 + .../spec/app/src/panel/sections/fill.ts | 32 + .../spec/app/src/panel/sections/layout.ts | 86 + .../spec/app/src/panel/sections/stroke.ts | 40 + .../spec/app/src/panel/sections/text.ts | 56 + .../spec/app/src/persist/deserialize.ts | 37 + .../spec/app/src/persist/migrations/index.ts | 40 + .../spec/app/src/persist/migrations/v1-v2.ts | 15 + .../spec/app/src/persist/migrations/v2-v3.ts | 25 + .../spec/app/src/persist/migrations/v3-v4.ts | 23 + .../spec/app/src/persist/migrations/v4-v5.ts | 13 + .../spec/app/src/persist/schema.ts | 74 + .../spec/app/src/persist/serialize.ts | 26 + .../spec/app/src/persist/viewstate.ts | 45 + .../proof-build/spec/app/src/plugin/api.ts | 50 + .../spec/app/src/plugin/capabilities.ts | 15 + .../proof-build/spec/app/src/plugin/events.ts | 22 + .../proof-build/spec/app/src/plugin/host.ts | 107 + .../spec/app/src/plugin/sandbox.ts | 51 + .../spec/app/src/selection/bounds.ts | 21 + .../proof-build/spec/app/src/selection/hit.ts | 28 + .../spec/app/src/selection/rules.ts | 33 + .../spec/app/src/selection/selection.ts | 62 + .../proof-build/spec/app/src/units/expr.ts | 124 + .../proof-build/spec/app/src/units/format.ts | 21 + .../proof-build/spec/app/src/units/parse.ts | 19 + .../spec/app/src/units/precision.ts | 30 + .../spec/app/src/units/registry.ts | 54 + .../proof-build/spec/app/src/util/assert.ts | 36 + .../proof-build/spec/app/src/util/emitter.ts | 49 + .../proof-build/spec/app/src/util/ids.ts | 39 + .../proof-build/spec/app/src/util/math.ts | 42 + .../proof-build/spec/app/test/canvas.test.ts | 75 + .../spec/app/test/commands.test.ts | 135 + .../spec/app/test/fixtures/doc-v5.json | 200 ++ .../spec/app/test/fixtures/v1.json | 45 + .../spec/app/test/fixtures/v4.json | 41 + .../spec/app/test/geometry.test.ts | 113 + .../proof-build/spec/app/test/group.test.ts | 56 + .../proof-build/spec/app/test/helpers.ts | 41 + .../proof-build/spec/app/test/history.test.ts | 104 + .../proof-build/spec/app/test/keys.test.ts | 90 + .../proof-build/spec/app/test/layers.test.ts | 92 + .../spec/app/test/migrations.test.ts | 47 + .../proof-build/spec/app/test/model.test.ts | 119 + .../proof-build/spec/app/test/palette.test.ts | 40 + .../proof-build/spec/app/test/panel.test.ts | 139 + .../proof-build/spec/app/test/persist.test.ts | 114 + .../proof-build/spec/app/test/plugin.test.ts | 120 + .../spec/app/test/precision.test.ts | 38 + .../spec/app/test/selection.test.ts | 101 + .../proof-build/spec/app/test/style.test.ts | 70 + .../proof-build/spec/app/test/units.test.ts | 63 + benchmarks/proof-build/spec/banned.mjs | 104 + benchmarks/proof-build/spec/filler.mjs | 555 ++++ benchmarks/proof-build/spec/history.mjs | 2432 +++++++++++++++++ benchmarks/proof-build/spec/proof.mjs | 2155 +++++++++++++++ .../proof-build/spec/versions/c84/README.md | 6 + .../spec/versions/d4/src/keys/dispatcher.ts | 143 + .../spec/versions/d4/src/keys/keymap.ts | 64 + benchmarks/proof-build/tasks/B-K1/task.json | 15 + benchmarks/proof-build/tasks/B-K2/task.json | 15 + benchmarks/proof-build/tasks/B-K3/task.json | 15 + benchmarks/proof-build/tasks/B-P1/task.json | 21 + benchmarks/proof-build/tasks/B-P3/task.json | 15 + benchmarks/proof-build/tasks/B-P5/task.json | 25 + benchmarks/proof-build/tasks/B-P6/task.json | 21 + benchmarks/proof-build/tasks/B-P7/task.json | 22 + benchmarks/proof-build/tasks/B-P8/task.json | 16 + benchmarks/proof-build/tasks/B-S1/task.json | 16 + benchmarks/proof-build/tasks/B-S2/task.json | 16 + benchmarks/proof-build/tasks/B-S3/task.json | 16 + benchmarks/proof-build/tasks/B-S4/task.json | 16 + benchmarks/proof-build/tasks/B-V1/task.json | 20 + benchmarks/proof-build/tasks/B-V2/task.json | 20 + benchmarks/proof-build/tasks/B-V3/task.json | 24 + benchmarks/proof-build/tasks/B-V4/task.json | 25 + benchmarks/proof-build/tasks/B-V5/task.json | 25 + .../proof-build/tasks/L-K1/plans/good.md | 7 + .../proof-build/tasks/L-K1/plans/plausible.md | 8 + .../proof-build/tasks/L-K1/plans/trap.md | 6 + benchmarks/proof-build/tasks/L-K1/rubric.json | 13 + benchmarks/proof-build/tasks/L-K1/task.json | 10 + .../proof-build/tasks/L-K2/plans/good.md | 7 + .../proof-build/tasks/L-K2/plans/plausible.md | 7 + .../proof-build/tasks/L-K2/plans/trap.md | 7 + benchmarks/proof-build/tasks/L-K2/rubric.json | 13 + benchmarks/proof-build/tasks/L-K2/task.json | 10 + .../proof-build/tasks/L-P1/plans/good.md | 8 + .../proof-build/tasks/L-P1/plans/plausible.md | 14 + .../proof-build/tasks/L-P1/plans/trap.md | 7 + benchmarks/proof-build/tasks/L-P1/rubric.json | 14 + benchmarks/proof-build/tasks/L-P1/task.json | 31 + .../proof-build/tasks/L-P2/plans/good.md | 7 + .../proof-build/tasks/L-P2/plans/plausible.md | 8 + .../proof-build/tasks/L-P2/plans/trap.md | 6 + benchmarks/proof-build/tasks/L-P2/rubric.json | 14 + benchmarks/proof-build/tasks/L-P2/task.json | 26 + .../proof-build/tasks/L-P3/plans/good.md | 7 + .../proof-build/tasks/L-P3/plans/plausible.md | 8 + .../proof-build/tasks/L-P3/plans/trap.md | 6 + benchmarks/proof-build/tasks/L-P3/rubric.json | 14 + benchmarks/proof-build/tasks/L-P3/task.json | 26 + .../proof-build/tasks/L-S1/plans/good.md | 7 + .../proof-build/tasks/L-S1/plans/plausible.md | 8 + .../proof-build/tasks/L-S1/plans/trap.md | 6 + benchmarks/proof-build/tasks/L-S1/rubric.json | 13 + benchmarks/proof-build/tasks/L-S1/task.json | 11 + .../proof-build/tasks/L-S2/plans/good.md | 7 + .../proof-build/tasks/L-S2/plans/plausible.md | 9 + .../proof-build/tasks/L-S2/plans/trap.md | 5 + benchmarks/proof-build/tasks/L-S2/rubric.json | 14 + benchmarks/proof-build/tasks/L-S2/task.json | 16 + .../proof-build/tasks/L-V1/plans/good.md | 7 + .../proof-build/tasks/L-V1/plans/plausible.md | 9 + .../proof-build/tasks/L-V1/plans/trap.md | 5 + benchmarks/proof-build/tasks/L-V1/rubric.json | 13 + benchmarks/proof-build/tasks/L-V1/task.json | 19 + 181 files changed, 17898 insertions(+) create mode 100644 benchmarks/proof-build/DESIGN.md create mode 100644 benchmarks/proof-build/README.md create mode 100644 benchmarks/proof-build/scripts/arms.mjs create mode 100644 benchmarks/proof-build/scripts/check-bed.mjs create mode 100644 benchmarks/proof-build/scripts/common.mjs create mode 100644 benchmarks/proof-build/scripts/compare.mjs create mode 100644 benchmarks/proof-build/scripts/gate.mjs create mode 100644 benchmarks/proof-build/scripts/gen-bed.mjs create mode 100644 benchmarks/proof-build/scripts/grade.mjs create mode 100644 benchmarks/proof-build/scripts/inject.mjs create mode 100644 benchmarks/proof-build/scripts/judge.mjs create mode 100644 benchmarks/proof-build/scripts/run.mjs create mode 100644 benchmarks/proof-build/spec/app/.gitignore create mode 100644 benchmarks/proof-build/spec/app/AGENTS.md create mode 100644 benchmarks/proof-build/spec/app/README.md create mode 100644 benchmarks/proof-build/spec/app/docs/plugins.md create mode 100644 benchmarks/proof-build/spec/app/src/canvas/gestures.ts create mode 100644 benchmarks/proof-build/spec/app/src/canvas/handles.ts create mode 100644 benchmarks/proof-build/spec/app/src/canvas/viewport.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/batch.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/align.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/delete.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/group.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/lock.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/move.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/nudge.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/rename.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/reorder.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/resize.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/set-prop.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/builtin/visibility.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/bus.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/coalesce.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/history.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/palette.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/registry.ts create mode 100644 benchmarks/proof-build/spec/app/src/commands/types.ts create mode 100644 benchmarks/proof-build/spec/app/src/editor.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/bindings/canvas.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/bindings/layers.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/bindings/panel.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/chords.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/dispatcher.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/keymap.ts create mode 100644 benchmarks/proof-build/spec/app/src/keys/scopes.ts create mode 100644 benchmarks/proof-build/spec/app/src/layers/drag.ts create mode 100644 benchmarks/proof-build/spec/app/src/layers/layers.ts create mode 100644 benchmarks/proof-build/spec/app/src/layers/rename.ts create mode 100644 benchmarks/proof-build/spec/app/src/layers/tree-view.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/document.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/factory.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/geometry.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/query.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/style.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/text.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/transform.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/tree.ts create mode 100644 benchmarks/proof-build/spec/app/src/model/types.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/commit.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/fields.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/mixed.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/properties.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/schema.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/sections/appearance.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/sections/fill.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/sections/layout.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/sections/stroke.ts create mode 100644 benchmarks/proof-build/spec/app/src/panel/sections/text.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/deserialize.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/migrations/index.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/migrations/v1-v2.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/migrations/v2-v3.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/migrations/v3-v4.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/migrations/v4-v5.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/schema.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/serialize.ts create mode 100644 benchmarks/proof-build/spec/app/src/persist/viewstate.ts create mode 100644 benchmarks/proof-build/spec/app/src/plugin/api.ts create mode 100644 benchmarks/proof-build/spec/app/src/plugin/capabilities.ts create mode 100644 benchmarks/proof-build/spec/app/src/plugin/events.ts create mode 100644 benchmarks/proof-build/spec/app/src/plugin/host.ts create mode 100644 benchmarks/proof-build/spec/app/src/plugin/sandbox.ts create mode 100644 benchmarks/proof-build/spec/app/src/selection/bounds.ts create mode 100644 benchmarks/proof-build/spec/app/src/selection/hit.ts create mode 100644 benchmarks/proof-build/spec/app/src/selection/rules.ts create mode 100644 benchmarks/proof-build/spec/app/src/selection/selection.ts create mode 100644 benchmarks/proof-build/spec/app/src/units/expr.ts create mode 100644 benchmarks/proof-build/spec/app/src/units/format.ts create mode 100644 benchmarks/proof-build/spec/app/src/units/parse.ts create mode 100644 benchmarks/proof-build/spec/app/src/units/precision.ts create mode 100644 benchmarks/proof-build/spec/app/src/units/registry.ts create mode 100644 benchmarks/proof-build/spec/app/src/util/assert.ts create mode 100644 benchmarks/proof-build/spec/app/src/util/emitter.ts create mode 100644 benchmarks/proof-build/spec/app/src/util/ids.ts create mode 100644 benchmarks/proof-build/spec/app/src/util/math.ts create mode 100644 benchmarks/proof-build/spec/app/test/canvas.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/commands.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/fixtures/doc-v5.json create mode 100644 benchmarks/proof-build/spec/app/test/fixtures/v1.json create mode 100644 benchmarks/proof-build/spec/app/test/fixtures/v4.json create mode 100644 benchmarks/proof-build/spec/app/test/geometry.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/group.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/helpers.ts create mode 100644 benchmarks/proof-build/spec/app/test/history.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/keys.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/layers.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/migrations.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/model.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/palette.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/panel.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/persist.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/plugin.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/precision.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/selection.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/style.test.ts create mode 100644 benchmarks/proof-build/spec/app/test/units.test.ts create mode 100644 benchmarks/proof-build/spec/banned.mjs create mode 100644 benchmarks/proof-build/spec/filler.mjs create mode 100644 benchmarks/proof-build/spec/history.mjs create mode 100644 benchmarks/proof-build/spec/proof.mjs create mode 100644 benchmarks/proof-build/spec/versions/c84/README.md create mode 100644 benchmarks/proof-build/spec/versions/d4/src/keys/dispatcher.ts create mode 100644 benchmarks/proof-build/spec/versions/d4/src/keys/keymap.ts create mode 100644 benchmarks/proof-build/tasks/B-K1/task.json create mode 100644 benchmarks/proof-build/tasks/B-K2/task.json create mode 100644 benchmarks/proof-build/tasks/B-K3/task.json create mode 100644 benchmarks/proof-build/tasks/B-P1/task.json create mode 100644 benchmarks/proof-build/tasks/B-P3/task.json create mode 100644 benchmarks/proof-build/tasks/B-P5/task.json create mode 100644 benchmarks/proof-build/tasks/B-P6/task.json create mode 100644 benchmarks/proof-build/tasks/B-P7/task.json create mode 100644 benchmarks/proof-build/tasks/B-P8/task.json create mode 100644 benchmarks/proof-build/tasks/B-S1/task.json create mode 100644 benchmarks/proof-build/tasks/B-S2/task.json create mode 100644 benchmarks/proof-build/tasks/B-S3/task.json create mode 100644 benchmarks/proof-build/tasks/B-S4/task.json create mode 100644 benchmarks/proof-build/tasks/B-V1/task.json create mode 100644 benchmarks/proof-build/tasks/B-V2/task.json create mode 100644 benchmarks/proof-build/tasks/B-V3/task.json create mode 100644 benchmarks/proof-build/tasks/B-V4/task.json create mode 100644 benchmarks/proof-build/tasks/B-V5/task.json create mode 100644 benchmarks/proof-build/tasks/L-K1/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-K1/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-K1/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-K1/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-K1/task.json create mode 100644 benchmarks/proof-build/tasks/L-K2/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-K2/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-K2/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-K2/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-K2/task.json create mode 100644 benchmarks/proof-build/tasks/L-P1/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-P1/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-P1/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-P1/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-P1/task.json create mode 100644 benchmarks/proof-build/tasks/L-P2/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-P2/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-P2/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-P2/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-P2/task.json create mode 100644 benchmarks/proof-build/tasks/L-P3/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-P3/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-P3/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-P3/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-P3/task.json create mode 100644 benchmarks/proof-build/tasks/L-S1/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-S1/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-S1/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-S1/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-S1/task.json create mode 100644 benchmarks/proof-build/tasks/L-S2/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-S2/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-S2/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-S2/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-S2/task.json create mode 100644 benchmarks/proof-build/tasks/L-V1/plans/good.md create mode 100644 benchmarks/proof-build/tasks/L-V1/plans/plausible.md create mode 100644 benchmarks/proof-build/tasks/L-V1/plans/trap.md create mode 100644 benchmarks/proof-build/tasks/L-V1/rubric.json create mode 100644 benchmarks/proof-build/tasks/L-V1/task.json diff --git a/benchmarks/proof-build/DESIGN.md b/benchmarks/proof-build/DESIGN.md new file mode 100644 index 00000000..c97137b5 --- /dev/null +++ b/benchmarks/proof-build/DESIGN.md @@ -0,0 +1,598 @@ +# Proof build benchmark: pre-registered design + +Status: pre-registered on 2026-10-20, before any bed, task or paid session +exists. Changes after the pilot are allowed only where this file says so, +and each one is logged at the end with its reason. + +## Question + +When an agent builds a feature from a product-level request, does putting +Proof records in its context make the work better or worse? Proof can help +when the right solution keeps a rule that only Proof records. It can hurt +when Proof is out of date, or when reading it costs attention that the code +needed. We compare six arms (no Proof, Proof on request, Proof forced in, +search before planning, search plus code paths, everything pasted in) on +the same code and the same requests, and score the results with hidden +tests (build tasks) and a blind rubric (plan tasks). + +Pre-registered hypotheses: + +- **H1 (help).** On proof-only tasks, `skill`, `find` and `findcode` are + all-green (acceptance, constraints and regressions all passing) more + often than `none`. A constraint pass counts only in a session that also + passed acceptance, since constraint tests skip when the feature is + missing. +- **H2 (redundant).** On code-visible tasks, no arm differs from `none` on + all-green. Proof only adds cost there. +- **H3 (harm).** On stale tasks, `brief` and `dump` do worse than `none` on + all-green. `skill`, `find` and `findcode` do no worse than `none`. +- **H4 (burden).** On control tasks, all-green is the same in every arm. + Context tokens, processed tokens and wall time rise with the amount of + Proof forced in (`none` ≤ `skill` < `find` ≤ `findcode` < `brief` < + `dump` < `dump10x`). These measures include the first turn, which holds + the pasted brief or dump; read tokens leave it out and are reported as + the session's own reading. + +## 1. The bed app: Tessera core + +Tessera is a fictional design editor. Its core is zero-dependency +TypeScript with no DOM, run by Node 22 with `--experimental-strip-types`. +It uses erasable syntax only: no enums, namespaces or parameter +properties, and imports use explicit `.ts` extensions. It has about 68 +source files and roughly 6,000 lines, plus about 16 test files and 1,600 +lines of `node:test` tests (`node --experimental-strip-types --test`, +which picks up `test/**/*.ts`; passing `test/` fails on Node 22.22). There +is no `package.json` dependency and no build step. + +`src/editor.ts#createEditor()` is the stable facade that every hidden test +drives: `doc`, `selection`, `history`, `panel` (`view()`, `edit(label, text)`), `layers`, `keys` (`press('Mod+Shift+H')`, `focus(scope)`), +`palette` (`run(title)`, `list()`), `plugins` (`load(manifest)`), and +`files` (`save()`, `load(json)`, `viewState()`). Hidden tests use only this +facade, palette titles, field labels, shortcut keys and plugin event names, +which are the names a user or plugin author sees. Because of that, any +reasonable implementation can pass. + +| Module | Holds | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/util/` (`ids`, `emitter`, `assert`, `math`) | id minting, a typed emitter, small math | +| `src/model/` (`types`, `document`, `tree`, `factory`, `geometry`, `transform`, `style`, `text`, `query`) | frames, rects, ellipses, lines, text and groups; parent/child operations; bounds; rotation (stored in radians since the drift commit, see §2); fills, strokes and opacity, with groups owning their own opacity since the drift commit; text runs | +| `src/units/` (`registry`, `expr`, `parse`, `precision`, `format`) | the unit registry (px, pt, in, %); the expression parser the panel uses (`expr.ts`, which handles arithmetic); `parse.ts`, a legacy wrapper now used only to read old string values in files; display precision per unit | +| `src/selection/` (`selection`, `hit`, `rules`, `bounds`) | the selection set; point hit-testing that skips hidden and locked nodes; selection bounds | +| `src/commands/` (`types`, `bus`, `history`, `coalesce`, `batch`, `registry`, `palette`) and `builtin/` (`move`, `resize`, `set-prop`, `reorder`, `group`, `delete`, `rename`, `visibility`, `lock`, `nudge`) | the command bus; undo/redo with a selection before and after each entry; gesture coalescing by gesture id; `batch()`, used only by `group`; palette titles | +| `src/panel/` (`properties`, `mixed`, `fields`, `commit`, `schema`) and `sections/` (`layout`, `fill`, `stroke`, `text`, `appearance`) | the properties-panel view model: sections, field descriptors, the `Mixed` sentinel, commit per field | +| `src/layers/` (`layers`, `tree-view`, `drag`, `rename`) | the layers-panel view model, listed frontmost first since the drift commit; expand and collapse; drag to reorder | +| `src/keys/` (`keymap`, `dispatcher`, `scopes`, `chords`) and `bindings/` (`canvas`, `layers`, `panel`) | the live bindings for each scope (`bindings/`, since the split); `keymap.ts` keeps `DEFAULT_KEYMAP`, a frozen legacy snapshot exported to plugins; focus scopes | +| `src/persist/` (`schema`, `serialize`, `deserialize`, `viewstate`) and `migrations/` (`index`, `v1-v2` … `v4-v5`) | an integer schema version (5); migrations applied one step at a time; a view-state sidecar (zoom, collapsed sections) kept out of the file | +| `src/plugin/` (`api`, `host`, `events`, `capabilities`, `sandbox`) | the frozen v1 plugin API (a header comment says frozen, and a surface snapshot test checks it); events that carry ids only; one capability (`export`) | +| `docs/plugins.md`, `AGENTS.md` | the plugin reference (names only, no rationale); how to run the tests | + +## 2. History and the Proof graph + +`scripts/gen-bed.mjs` builds the bed deterministically from +`spec/history.mjs` (ordered commits, each one a set of whole-file writes +with a fixed author, date and message) and `spec/proof.mjs` (the records, +as mutations replayed through the real `createProofWriter`, the way +`proof-recall/scripts/build-repo.mjs` does it). + +- **Commits.** 84 commits, from 2026-02-02 to 2026-06-12, by four authors: + Ines Okafor (model, persistence), Tomas Lindqvist (commands, history), + Priya Raman (panel, units) and Mateo Alves (layers, keys, plugins). A + fifth author, Ren Ito, joins in May and makes the drift commits. About 36 + commits implement a record and carry `Proof: ` trailers. The others + are fixes, tests and chores with ordinary subjects. Commit messages never + state a rationale that only Proof holds. +- **Variants.** `--variant proof` writes `.flatbread-proof/`, + `flatbread.config.js` and trailers. `--variant none` writes the same + commits with no trailers and no Proof files. The two variants have + identical code trees, and `check.mjs` verifies that with `git ls-tree`. + Hashes differ between variants. Citations pin hashes from the proof + variant. +- **Efforts.** There are seven, one for each area: Document model and + geometry; Units and precision; Command bus and history; Properties panel; + Selection, layers and shortcuts; Persistence and migrations; Plugin API + v1. Each has about 20 live records (around 5 Decisions with rationale + and rejected alternatives, 2 Constraints, 3 Findings, 2 Issues mostly + resolved, 1 or 2 Risks, 4 to 6 Citations `path#symbol@sha`, and one + compaction) plus 3 or 4 dead ones (superseded, rejected, invalidated, + retracted). That is about 140 live and 25 dead records. Two compactions + are stale (`compaction.stale`). +- **Proper supersession (Proof is right).** Coalescing by a 500 ms time + window was superseded by coalescing by gesture id, with the rationale + that keyboard nudges and slow drags merged wrongly. The undo cap went + from 100 to 200. +- **Drift (Proof is wrong, code is the truth).** Five late commits with no + trailer, none of which updates Proof: + + | Key | Kind | Proof still says | Code now | + | --- | ----------------------- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | + | D1 | unrecorded supersession | "Groups carry no style; opacity on a group applies to its children" | groups have their own `opacity`, multiplied at render; tests updated | + | D2 | unrecorded supersession | "Rotation is stored in degrees" | radians; the panel converts | + | D3 | unrecorded supersession | "The layers list is back-to-front, index 0 is the backmost layer" | frontmost first; drag indexes reversed | + | D4 | refactor drift | Citations and a Decision name `keys/keymap.ts#DEFAULT_KEYMAP` as where shortcuts are registered | bindings live in `keys/bindings/*`; `DEFAULT_KEYMAP` is a frozen snapshot the dispatcher never reads | + | D5 | refactor drift | Citations and a Decision name `units/parse.ts#parseLength` as the parser for panel input | the panel uses `units/expr.ts#parseExpression`; `parseLength` only reads legacy file strings | + +- **Rules planted in Proof.** Each rule is a Decision or Constraint with + its rationale. Most also have an Issue or Finding that motivated them + and a Citation. + + | Key | Rule | Visible in code or tests? | + | --- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | + | R1 | one user gesture is one history entry, however many nodes it touches (Issue: "undo needed 12 presses after a multi-select edit") | no (`batch()` exists, unexplained, and only `group` uses it) | + | R2 | command payloads are plain JSON holding node ids, never node objects (macro recording, replay) | in effect (the `Json` payload type, `commands.replay`, `groupNodes`) | + | R3 | a shipped migration is never edited; a schema change adds a new step (Issue: files diverged after an edited step) | no | + | R4 | coalesce by gesture id, never by a time window (the head of the supersession above) | in effect (the gesture-id API, `KeyEvent.repeat`, a test naming it) | + | R5 | plugins receive frozen snapshots, never live nodes (Issue: a plugin mutated nodes and bypassed history) | partly (event payloads are frozen at the top level; copies, deep) | + | R6 | view state (expanded groups, zoom) never goes into the document file; it goes in the sidecar | no (the sidecar exists, with no comment) | + | V1 | hidden and locked nodes are never hit or selected from the canvas | yes (the filter in `hit.ts` and a test) | + | V2 | stored geometry is never rounded; precision applies only to display | yes (a comment in `precision.ts` and a test that ten 0.1 nudges equal 1) | + | V3 | undo restores the exact selection from before the change | yes (the entry shape in `history.ts` and a test for `group`) | + | V4 | plugin API v1 is frozen | yes (a header comment and a surface snapshot test) | + +- **Filler at 10x** (`--scale 10`, used only by `dump10x`). Nine filler + Efforts in look-alike areas (export, prototyping, comments, assets, + colour management, collaboration cursors, onboarding, telemetry, fonts) + bring the graph to about 1,500 records. They are written in the same + voice and cite only filler paths that do not exist in the bed. They never + state a rule about the seven real areas. The code is identical. +- **No anticipation.** Records describe past work and standing rules. An + offline check fails the build if any record body, title or commit message + contains a task's feature phrases (for example "corner radius" or + "marquee"). Area words such as "properties panel" or "undo" are allowed. + +## 3. Tasks (26: 18 build, 8 plan) + +Prompts describe intended behaviour in a user's words. They name no file, +symbol, module, Effort or record id. They may name product surface: palette +titles, panel field labels, shortcut keys, and plugin event names as +`docs/plugins.md` lists them. Each task is +`p4/tasks//{task.json, acceptance.test.ts, constraints.test.ts, reference.patch, naive.patch}`, outside the repository. `task.json` holds +the id, kind, category, prompt, rule keys, `leakWords` (phrases that must +not appear in the none variant) and `featureWords` (phrases that must not +appear in Proof). + +### Build tasks (18) + +The list as it stands after the review before any run; the change log +says what changed from the pre-registered list and why. + +| Id | Category | Request (gist; the real prompt is 3 to 6 sentences of behaviour) | Rule | Naive patch | +| ---- | ------------ | -------------------------------------------------------------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------ | +| B-P1 | proof-only | Rename layers: a base name numbers the selected layers in selection order; `#` marks where the number goes; undoable | R1 | one rename command per layer, so one Undo per layer | +| B-P3 | proof-only | A Layout grid field on frames (Off or a size), saved with the file; files of any older version open with the grid off | R3 | a load-time default for the new field, with no new step or version | +| B-P5 | proof-only | A `selectionchange` plugin event with `{ layers }` (id, name, type, x, y, width, height) in selection order | R5 | plain copies in a payload frozen only at the top, like the existing emitters | +| B-P6 | proof-only | After reopening a document, the layers panel shows the same groups and frames expanded | R6 | an `expanded` flag on group and frame nodes, saved in the document file | +| B-P7 | proof-only | Copy style and Paste style: fills, strokes and opacity from one layer onto the selection; groups left alone; undoable | R1 | one set-prop command per property and layer, so a paste takes many Undos | +| B-P8 | proof-only | Old files that name colours (red, navy, …) open with hex colours, including files saved again since | R3 | the names mapped inside the shipped v2→v3 step, plus a load-time fix, with no new version | +| B-V1 | code-visible | Dragging on empty canvas draws a selection box; covered groups are taken whole, partly covered ones give up their covered layers | V1 | the walk skips the hit filter, so hidden and locked layers are selected | +| B-V2 | code-visible | Scale selection by a typed percentage about the selection's centre, contents included | V2 | stored values rounded to 2 decimals | +| B-V3 | code-visible | Ungroup (palette and Mod+Shift+G): children take the group's place and become selected; one Undo restores the groups | V3 | the freed layers are selected before the batch, so Undo does not reselect the groups | +| B-V4 | code-visible | Duplicate (palette and Mod+D): copies 10 px right and down, in front of the originals, selected; Undo and Redo | R2, V3 | the copies are selected after the batch, so Redo brings them back unselected | +| B-V5 | code-visible | Holding an arrow key keeps nudging; one Undo takes back the whole hold; letting go ends the step | R4 | repeats within 400 ms by the editor clock merge, so a hold whose repeats pause splits | +| B-S1 | stale (D1) | Number keys 1–9 and 0 set the selected layers' opacity | D1 | follows Proof: a group's value is written to the layers inside it | +| B-S2 | stale (D2) | Shift while rotating snaps to 15° steps; the Rotation field shows the snapped angle | D2 | follows Proof: the snap is done and stored in degrees | +| B-S3 | stale (D4) | Mod+Shift+H hides or shows the selected layers; the step is labelled Hide or Show | D4 | follows Proof: the chord is also added to `DEFAULT_KEYMAP`, changing the frozen v1 table | +| B-S4 | stale (D5) | W and H accept in, pt, px and % of the enclosing frame, mixed with arithmetic | D5 | follows Proof: `parseLength` is extended and tried first, so a unit inside arithmetic is refused | +| B-K1 | control | Read-only Characters and Words fields in the Text section | none | the fields in a new section, which breaks an existing test | +| B-K2 | control | Fill and Stroke accept `#f80`, six-digit hex with or without `#`, and `rgb(r, g, b)`; shown as upper-case `#RRGGBB` | none | `parseColor` itself widened, which breaks an existing test | +| B-K3 | control | Distribute horizontally or vertically with an optional typed gap | none | a missing gap read as 0, which breaks an existing test | + +### Plan tasks (8) + +The agent writes `PLAN.md` and changes no code. Each plan task needs at +least two rules and is scored on a rubric (§5). + +| Id | Category | Request (gist) | What it hinges on | +| ---- | ------------ | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| L-P1 | proof-only | Components: turn a frame into a reusable main; instances update; text and fill overrides | R1 (an override edit is one entry), R3 (a new migration step), R5 (plugins get snapshots of instances), V4 (no v1 fields) | +| L-P2 | proof-only | Version history: browse and restore earlier versions of a document | R2 (replayable commands), R6 (view state excluded), R3 | +| L-P3 | proof-only | Auto layout: frames stack children with gap and padding | R1 (relayout belongs to the edit's entry), R4, V2 | +| L-V1 | code-visible | Snapping to other layers' edges and centres while dragging | V1 (no snapping to hidden or locked layers), V2 (snap shown, not rounded in storage) | +| L-S1 | stale | Masks: a shape clips the layers above it in its group | D3 (frontmost-first order), D1 (groups have their own opacity); must not plan on back-to-front indexes | +| L-S2 | stale | Users can rebind shortcuts in preferences | D4 (rebinding goes through `bindings/` and the dispatcher; `DEFAULT_KEYMAP` stays frozen) | +| L-K1 | control | Rulers along the top and left with nicely spaced labelled ticks for the current zoom | none | +| L-K2 | control | A document swatches list built from the fills in use, sorted by frequency | none | + +## 4. Grading build tasks + +1. **Capture.** When a session ends, the runner writes `session.diff`: + `git add -A` and `git diff --cached HEAD`, leaving out + `.flatbread-proof/`, `.claude/`, `AGENTS.md`, `CLAUDE.md`, + `flatbread.config.js` and the runner's own files. It then deletes the + clone to save disk. +2. **Apply.** The diff is applied with `git apply --3way` to a pristine + worktree of the none variant at `p4/grade//`, which is reset and + cleaned before each session. So every arm is graded on byte-identical + code. If the diff does not apply, every check fails and `apply_failed` + is recorded. +3. **Restore.** Everything the bed tracks under `test/` (test files, + `helpers.ts`, fixtures) is restored from the pristine tree, with renames + seen as a deletion plus an addition. An agent that edited an existing + test is flagged `edited_existing_tests`, one that edited a helper or + fixture `edited_test_support`, and the originals still decide + regressions. +4. **Run.** Hidden tests are copied in only now, to `test/_hidden/`. Three + separate `node --experimental-strip-types --test` runs follow, each with + a 120 s timeout, with no network (`unshare -n`, where the machine allows + it) and no proxy or session variables: + - **Regressions:** the original suite. + - **Acceptance:** `acceptance.test.ts`, the behaviour asked for. + - **Constraints:** `constraints.test.ts` plus the global invariants. + `global.test.ts` runs on every task: a document saves, loads and saves + the same text, and plugin API v1 keeps its members and shortcut table + (both stated in the code). `global-rules.test.ts` runs on proof-only + and stale tasks only: every shipped migration step returns what it + always returned, and every palette command makes at most one history + entry per run. Those are R3 and R1, which only Proof states, so a + control or code-visible task is never graded on them. + The agent's own new tests run as well and are recorded + (`agent_tests_pass`) but not scored. `tsc --noEmit`, from the repo's + node_modules, is recorded as `typecheck` and is not scored. +5. **Report per task.** `acceptance`, `constraints`, `regressions` and + `all_green` (all three pass), plus the failing test names. + +**What each constraint test checks, by category.** Proof-only: the R rule +(for example, one `undo()` reverts every node in B-P1; a `JSON.stringify` +round-trip of the recorded command replays the same result in B-P2). +Code-visible: the V rule. Stale: the code's truth (a group's own opacity +changes and its children's opacity does not; rotation is in radians; the +frozen `DEFAULT_KEYMAP` snapshot is unchanged; `parseLength` behaves as +before on legacy strings). Control: global invariants only. + +**Offline gate** (`scripts/gate.mjs`, free, run before any paid session; +it must pass for every task): + +- The base tree fails acceptance and passes regressions and constraints. +- The reference patch passes all three. +- The naive patch passes acceptance and regressions, and fails only the + test file its category predicts. For stale tasks, the naive patch is the + "follow Proof" solution. B-S3 and B-S4 are the exception: their naive + patch fails acceptance, which is the predicted failure there. For + control tasks, the naive patch fails regressions only, and a second + "Proof-flavoured" reference that also keeps every R rule must pass + everything, which shows the constraint tests do not reward style. +- `leakWords` for each R rule appear nowhere in the none variant's code, + tests, docs or commit messages. The phrases for each V rule do appear. +- `featureWords` appear nowhere in Proof. +- The two variants' code trees are identical. + +## 5. Grading plan tasks + +- **Rubric.** Each plan task has a rubric in `p4/tasks//rubric.json`: + 4 to 6 must-include points and 2 to 4 must-avoid points, each a single + checkable claim (for example "edits to an instance override are + undoable in one step", or "must avoid: storing expand state on nodes in + the file"). Rubrics may name code modules, because plans will. +- **Blinding.** Before judging, `PLAN.md` is stripped of record ids (with + or without their hash suffix, and as qualified keys such as + `model.dec-group-nostyle`), `./flatbread proof` commands, `.flatbread-proof` + paths, the words "Proof" (not "future-proof" or "proof of") and "project + memory", phrases such as "the recorded decision" or "decision record", + and "Effort" and "compaction". Each is cut out together with the + parenthesis, bracket or lead-in ("per …", "see …", "… says:") that held + it, and nothing marks the cut, so a Proof-arm plan cannot be told from a + `none` plan by its markers and keeps the rest of each line. The judge is + told to score content only, whatever its source. +- **Judge.** A fixed judge, `claude-opus-5-5` at `--effort high` with no + tools, gets the stripped plan, the request and the rubric on stdin. It + returns JSON with met or violated for each point and a short quote. It + runs twice per plan. A point counts when both runs agree; when they + disagree, the point counts as half and is listed. +- **Score.** `(includes met + avoids not violated) / (|I| + |A|)`, from 0 + to 1. Must-avoid violations are also reported on their own as plan + constraint violations. +- **Check.** `scripts/judge.mjs --prepare-only` writes the judge inputs + with no paid call. The rubric gate (`judge.mjs --gate`) calls the judge, + so it runs in the pilot, before the full run. Each plan task has three + hand-written plans: a good one, which must score at least 0.8; a trap one + that walks into the stale or rule trap, which must score at most 0.5; + and a plausible one, a realistic plan written from the code alone as a + `none` session would, whose score is reported. On a proof-only task a + plausible plan that scores as high as the good one means the rubric does + not reward what only Proof says, and the rubric is revised before the + full run (one of the pilot's allowed changes). Control rubrics reward + only what the request and the code ask for, never an R rule. + +## 6. Arms + +The code and the request are identical in every arm. The whole task text +goes to `claude -p` on stdin, never as a file to read, because a pasted +brief or dump is larger than Linux's 128 KB limit on one argument and +larger than what `Read` returns. Every session is a fresh `claude -p` with +these settings: + +- `--model` and `--effort` recorded from the init event. +- `--strict-mcp-config` with no MCP servers. +- `--setting-sources project`, so a skill in the clone loads. +- Tools Bash, Read, Write, Edit, Glob, Grep and Skill in every arm. +- No session persistence and a 1,500 s timeout. + +Each session gets its own `git clone --no-hardlinks` with the remote +renamed and reflogs removed, as `prepareClone` does. The runner forks +`proof-recall/scripts/run.mjs` and keeps `--prepare-only`. +`FLATBREAD_PROOF_SERVER` is never set. Reads run locally against the +clone's own graph. + +| Arm | Clone | Added to the task text | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `none` | none variant; `AGENTS.md` (tests, style) | nothing | +| `skill` | proof variant; `.claude/skills/proof/` (a copy of `.agents/skills/proof/`); `./flatbread` wrapper; `AGENTS.md` plus one line: "Project memory (decisions, constraints, findings) lives in Proof; read it with `./flatbread proof …` (see the proof skill)"; `CLAUDE.md` = `@AGENTS.md` | nothing | +| `brief` | as `skill` | the output of `proof brief ` for each of the 7 Efforts, under "Project memory briefs" | +| `find` | as `skill` | "Before planning, run `./flatbread proof find --all`, then `proof get` any hit that bears on the request." | +| `findcode` | as `skill` | the `find` text plus: "Before editing a file, run `./flatbread proof find --path `; before relying on a Decision, run `proof find --refs all`." | +| `dump` | as `skill` | every live record body (all kinds, compactions included), grouped by Effort | +| `dump10x` | as `skill`, on the 10x graph | as `dump`, 1,074 live records of 1,547 (the burden ceiling; an extra arm, not a primary comparison) | + +`none` has no `CLAUDE.md` line about Proof, no skill, no wrapper, no +config and no node_modules link. The proof arms link node_modules (for the +CLI) through `.git/info/exclude`. Every arm's task text ends with the same +closing line: for build tasks "Implement it, add tests, and run the test +suite", and for plan tasks "Write PLAN.md in the repository root; do not +change code." + +## 7. Metrics and analysis + +- **Per session.** `all_green`, `acceptance`, `constraints` and + `regressions`; the number of failing constraint tests; plan score and + must-avoid hits. A constraint pass is counted only in a session that + also passed acceptance (`constraints`); the plain count, which an empty + diff passes, is kept as `constraints_raw` for reference. Cost comes from + `proof-recall/scripts/collect.mjs`: first-turn tokens (system prompt + plus task text, so the pasted brief or dump shows here), context tokens + (every fresh token, first turn included), read tokens (context tokens + without the first turn), processed tokens (context plus cache reads), + tool calls and turns. Wall time comes from the result event. Also + recorded: Proof reads (count of `./flatbread proof` calls), Proof writes + (any mutation), direct reads of `.flatbread-proof/` files, searches whose + results showed Proof files (`proof_search_exposure`: a repository-wide + Grep reaches records in every Proof arm, `skill` included), whether the + session opened a cited path, and `apply_failed`, `edited_existing_tests`, + `edited_test_support`, `timed_out` and `api_error`. +- **Runs.** Build tasks run 3 times per arm, plan tasks twice; `dump10x` + runs once. +- **Primary endpoints.** All-green on the proof-only build tasks (H1, + 6 tasks × 3 runs = 18 pairs per arm) and on the stale build tasks (H3, + 4 × 3 = 12 pairs). Each of the five arms (`skill`, `brief`, `find`, + `findcode`, `dump`) is compared with `none` in each of the two + categories by a two-sided exact sign test over the discordant + (task, run) pairs; the ten p values are Holm-corrected together at + α = 0.05 (`holm_primary` in `compare.mjs`). The stricter task-level sign + test, which counts each task once as in `proof-recall/scripts/compare.mjs`, + is reported beside each. `dump10x` is compared with `dump`, without + correction. +- **Secondary.** All-green pooled over all 18 build tasks, Holm-corrected + across the five arms (`holm_all_green`); it mixes categories whose + predicted effects run in opposite directions, so it is read only with + the category results beside it. Code-visible and control all-green + (H2, H4). Plan score, compared with a paired Wilcoxon signed-rank test + per arm against `none`. Context tokens, processed tokens and wall time + per arm, by median and paired ratio (H4). Every result is also reported + per category. +- **Leakage controls.** A session is flagged and excluded, together with + its pair across arms for that task and run, when any of these holds: + `left_folder` (a file tool, Glob, Grep or shell path outside the clone; + heredoc bodies and inline scripts count only for absolute paths under a + protected folder, so code written through the shell is not a leak); + any path under `p4/` or the benchmark source appears in a tool input + (`hidden_read` for the task, hidden and grading folders and the gate + log); or a `none` session found Proof by another route, which should be + impossible. The report gives flag counts by arm. Sessions that time out + stay in and count as failures. A session that ends in an API error is + not a task failure: transient errors are retried (up to 3 attempts), + and a (task, run) with any other API error is dropped from every arm and + counted per arm (`compare.mjs --errors fail` keeps them as failures, to + read a context overflow in `dump10x` as burden). + +## 8. Cost + +Rates are first-party API rates for `claude-haiku-5-5`: $0.10 per million +input tokens and $0.50 per million output tokens up to a 100K-token +prompt, and $0.50 and $2.50 above it. Cache reads cost 0.1× input and cache +writes 1.25× input. For `claude-sonnet-5-5`: $2 and $10, with cache reads +at $0.20. The judge is `claude-opus-5-5` at $4 and $20. + +The session sizes below are assumptions until the pilot measures them. +Past `proof-recall` runs read 0.3 to 0.6 M processed tokens per question, +and editing work costs more. The pasted text measured in prepare-only +runs is about 0.8 to 1.3 K characters for `none`, `skill`, `find` and +`findcode`, 24 K for `brief`, 75 K for `dump` and 481 K (about 120 K +tokens) for `dump10x`. + +| Session shape | Turns | Processed tokens | Cache writes | Output | Haiku cost | +| --------------------------------------- | ----- | ---------------- | ------------ | ------ | ---------- | +| build, `none` / `skill` / `find` | ~45 | ~2.0 M | ~120 K | ~20 K | ~$0.05 | +| build, `brief` (+~25 K fixed) | ~45 | ~3.1 M | ~150 K | ~20 K | ~$0.06 | +| build, `dump` (+~50 K fixed) | ~45 | ~4.3 M | ~180 K | ~20 K | ~$0.08 | +| build, `dump10x` (+~450 K, above 100 K) | ~45 | ~22 M | ~0.6 M | ~20 K | ~$1.50 | +| plan, any arm except `dump10x` | ~25 | ~1.0 M | ~80 K | ~8 K | ~$0.03 | + +- **Pilot** (one build task per category: B-P1, B-S3, B-V4, B-K1; the six + primary arms; 1 run; Haiku). 24 sessions, about $1.40 (range $1 to $4). + The rubric gate (24 hand-written plans, judged twice by Opus, 48 calls) + costs about $3. Two optional add-ons: `dump10x` on B-K1 (1 session, + about $1.50, to see that a 120 K-token prompt runs at all) and L-P1 in + the six arms (6 sessions and 12 judge calls, about $1, the only way to + exercise PLAN.md capture and the judge on real sessions before the full + run). The pilot must clear these gates before the full run: + + - every session finishes, with no API error left after retries; + - the init event lists the `proof` skill in the proof arms; + - `find` and `findcode` call `find` in at least 3 of 4 sessions; + - no leakage flag shows a harness bug; + - grading reproduces the gate's verdicts on the reference patches, + re-applied as session diffs; + - the rubric gate passes (good ≥ 0.8, trap ≤ 0.5); on a proof-only + task, a plausible plan that scores as high as the good one marks a + rubric to fix. + + The pilot may change prompts' wording, the timeout, the stripping rules + and, where the rubric gate shows it, a rubric's points, and nothing + else. Changes are logged below. + +- **Full run on Haiku** (18 build tasks × 6 arms × 3 runs = 324 sessions + and 8 plan tasks × 6 arms × 2 runs = 96, plus `dump10x` once per task, + 26). The 420 primary sessions cost about $20 to $28 and the 26 `dump10x` + sessions about $30 to $40. Judging covers 104 plans, twice each, for + about $14. The total is about $65 to $85. At 8 parallel jobs and about + 8 minutes per session, that is about 7.5 hours of wall time. +- **Sonnet subset** (the 8 build tasks P1, P3, P5, P7, V1, S1, S3 and K1; + the arms `none`, `skill`, `find` and `brief`; 1 run). 32 sessions at + about $0.90 each, for about $30. It is read only as a check on the + direction of the Haiku result. +- **Disk.** Clones are about 5 MB each and are deleted after capture. What + is kept for each session is `session.diff`, `PLAN.md`, the gzipped + transcript (about 0.3 MB) and the grade JSON, roughly 160 MB for the + full run. + +## Layout + +`benchmarks/proof-build/` holds: + +- `spec/history.mjs` and `spec/proof.mjs`: the bed's commits and records. +- `spec/filler.mjs`: the 10x filler. +- `tasks//`: each task's `task.json`, and for plan tasks `rubric.json` + and `plans/`. +- `scripts/` (`gen-bed.mjs`, `check-bed.mjs`, `gate.mjs`, `run.mjs`, + `grade.mjs`, `judge.mjs`, `compare.mjs`, with `arms.mjs`, `inject.mjs` and + `common.mjs`), reusing `proof-recall`'s `collect.mjs` and its sign test. + +`p4/` (outside the repo) holds: + +- `bed/`, `bed-none/`, `bed-x10/` and their key maps +- `hidden//` (hidden tests and patches) and `hidden/_global/` +- `tasks//` (assembled by `gate.mjs`) +- `grade/` +- `runs/` + +## Change log + +- 2026-10-20: pre-registered. +- 2026-10-20, task authoring, before any paid session: + - **Bed tests relaxed.** `test/plugin.test.ts` now pins the first three + plugin events instead of the whole list, since Proof allows new v1 + events and the old pin made every B-P5 solution fail regressions. + `test/persist.test.ts` checks the restored zoom and collapsed sections + field by field instead of comparing the whole view state, which made + every B-P6 solution that adds to the view state fail. The beds were + rebuilt (tips: bed `7a02228`, bed-none `ad36fdf`, bed-x10 `beda051`) + and `check-bed.mjs` passes. + - **Gists changed where the bed made the planned one useless.** B-P1 is + now Rename layers (numbered names, one undo), because field commits + already share a gesture id, so a corner-radius field could not break + R1. B-S1 is now number keys that set opacity, because the Opacity field + at the tip already handles groups, so the base tree passed the planned + task. B-K3 is now Distribute with a gap, because a layers filter needs a + facade method the prompt would have to name. B-P2 drops "continues the + same offset" (Mod+D on the copies moves them another 10 px). B-P3's + field is Layout grid (Off or a size). + - **Naive patches.** For B-S2 the Proof-following patch (snap and store + degrees) fails acceptance as well as constraints, because the Rotation + field shows the stored number read as radians; B-S4's fails acceptance + only, as planned. Control naive patches are plausible solutions that + break an existing test, so they fail regressions. No second + "Proof-flavoured" control reference was written. + - **Constraint tests** check their rule only where the build has the + feature, so the base tree passes them. + - **Global invariants** (`_global/global.test.ts`): every shipped migration + step returns what it returned at the tip on fixed inputs (including a + version 1 file with unit strings); a document saves, loads and saves the + same text (instead of the version 5 fixture round-trip, since a correct + schema change bumps the version); every palette command makes at most one + history entry per run, with and without input, over four selections; and + plugin API v1 keeps its members and its shortcut table. + - **Layout.** `task.json`, `rubric.json` and the good and trap plans are in + `benchmarks/proof-build/tasks/`; hidden tests and patches are in + `p4/hidden//`; `scripts/gate.mjs` (the `check.mjs` of §4) assembles + `p4/tasks/` and runs the offline gate. +- 2026-10-21, after the design and harness reviews, before any paid + session: + - **H1 is scored on all-green.** Constraint tests skip when the feature + is missing, so counting constraint passes alone let a session that + built nothing win. A constraint pass now counts only with acceptance, + and H1 reads all-green. + - **Primary endpoints per category.** The pooled all-green mixes + proof-only tasks (predicted help) with stale ones (predicted harm), so + proof-only and stale all-green are now co-primary, five arms each, + Holm-corrected together over the ten tests; the pooled result is + secondary. Build tasks run three times instead of twice. + - **Proof-only tasks that did not need Proof.** B-P2 (Duplicate) and B-P4 + (hold an arrow key) are now code-visible, as B-V4 and B-V5: the `Json` + payload type, `commands.replay` and `groupNodes` show R2, and + `KeyEvent.repeat`, the unused `release()` hook, `ctx.newGesture()` and + the test "commands without a gesture id stay apart however close in + time" show R4. B-V4's naive patch is now a realistic mistake (selecting + the copies after the batch). B-V5's prompt no longer echoes a record + ("separate presses stay separate"). Two proof-only tasks replace them: + B-P7 (Copy style and Paste style, R1) and B-P8 (old files that name + colours, R3). + - **B-P3 could be passed without Proof.** A load-time default with no new + step passed every test, and the accepted record + `persist.dec-defaults-on-load` said "Migrations do not have to add + every field", against `persist.con-migrations-frozen`. The record now + says load defaults cover files from before versions existed and are + not how the format gains a field; a new constraint test requires a + newer saved version reached by a step from version 5; the naive patch + is now the load-time default alone. The L-P1 and L-P3 rubrics, which + ask for a new step, now agree with Proof. + - **B-P6 gave its answer away.** The prompt no longer mentions other view + settings, its acceptance test for them is gone (the bed's own tests + cover them), and `persist.dec-viewstate-sidecar` no longer lists + "which groups the layers panel shows expanded". + - **B-P5: Proof and code disagreed.** `plugin.dec-ids-only-events` now + says events carry ids and plain copied values, never nodes, instead of + "a plugin that needs more asks for a capability". The plugin event + payloads are now frozen where they are emitted (commit c33, "Remove + getNode from the plugin API"), as `plugin.con-frozen-snapshots` says; + the freeze is shallow, so deep freezing and copies stay Proof-only. The + naive patch is now plain copies in a top-level frozen payload, which is + what copying the existing emitters gives. + - **Prompts that steered.** B-S1 no longer says "exactly what typing that + percentage into the Opacity field does, … groups included"; B-S3 no + longer points at Mod+Shift+L, which sits in both the canvas bindings + and `DEFAULT_KEYMAP`. + - **Rules graded where they cannot be seen.** The global invariants are + split: R1 and R3 (`global-rules.test.ts`) run only with proof-only and + stale tasks; stable saving and plugin API v1 (`global.test.ts`) run + with every task. + - **Control plan rubrics rewarded Proof rules.** L-K1 and L-K2 no longer + have points about the document file, history or the command bus; their + trap plans match the new points. + - **Rubric gate.** Each plan task gains `plans/plausible.md`, a realistic + plan written from the code alone. `judge.mjs --gate` judges good, trap + and plausible plans; it runs in the pilot, since it calls the judge. + - **Bed.** `test/persist.test.ts`'s fixture round-trip now checks that + every field of the version 5 fixture survives a save, rather than the + whole file, so a correct schema change does not fail regressions and + grading can restore fixtures. The filler topic "colour names" is now + "colour pickers", and the new tasks' feature words are banned in + Proof. Rebuilt tips: bed `598221b`, bed-none `bdb60f4`, bed-x10 + `06b0e6d`; `check-bed.mjs` passes. + - **Harness.** The leak scan no longer flags `..`, `/` or `//` inside + heredoc bodies and inline scripts, and watches the whole `p4/` folder + (hidden tests, gate log) in Bash too. Cost reports first-turn and + context tokens, so the pasted brief and dump count toward H4. Plan + blinding cuts Proof wording with its clause instead of leaving `[ref]` + markers that only Proof-arm plans would carry. API errors are retried + or reported apart. Renamed test files and edited helpers or fixtures + no longer slip past grading. Grading runs without network where + `unshare -n` works. `run.mjs` defaults match the `p4/` layout, and its + printed grade command passes `--bed`. +- 2026-10-21, offline gate before the pilot, before any paid session: + - **Beds rebuilt from scratch** match the previous builds object for + object (commit and tree hashes, key maps, working trees), so the build + is deterministic. `check-bed.mjs` passes 33 checks and the suite 132 + of 132. + - **Blinding touched plans it should not have.** `stripPlan` collapsed + indentation, dropped `()` after function names and removed trailing + spaces in every plan, so 9 of the 24 hand-written plans changed with + no Proof wording in them. It now edits only lines that held a cut, so a + plan with no Proof wording comes out byte for byte the same. It also + cut `an ` out of words such as "ran" ("I ran `./flatbread proof find …`" became "I r and read …"), left a dangling "as." where "as the + project memory records" was cut, and left empty backticks round a cut + id. A sentence that only reports running a Proof command now goes + whole, a lead-in left pointing at nothing goes, a code span holding + only cut ids goes, and a line whose opening words were cut starts + upper case. + - **Pilot tasks.** The pilot now takes one build task per category + (B-P1, B-S3, B-V4, B-K1) in the six primary arms, 24 sessions; L-P1 + and `dump10x` are optional add-ons. The `dump10x` arm text gives its + real size: 1,074 live records of the 1,547 record files. diff --git a/benchmarks/proof-build/README.md b/benchmarks/proof-build/README.md new file mode 100644 index 00000000..dfecd2fc --- /dev/null +++ b/benchmarks/proof-build/README.md @@ -0,0 +1,170 @@ +# Proof build benchmark + +When an agent builds a feature from a product-level request ("the +properties panel should also…"), does having Proof records in its context +make the work better, worse, or just more expensive? The pre-registered +design, hypotheses and change log are in [DESIGN.md](DESIGN.md). + +## What it measures + +The test bed is Tessera core, a small zero-dependency TypeScript design +editor with 84 commits of history. In the proof variant, 7 Efforts of +records written the way real agents write them (Decisions with rejected +alternatives, Constraints, Findings, Risks, code Citations) sit in the +graph. The none variant has the same code and no Proof at all. The 10x +variant adds 9 filler Efforts, about ten times the records. + +There are 26 tasks: 18 build tasks graded by hidden tests and 8 plan tasks +scored by a blind rubric. Each task belongs to one category: + +| Category | What makes a solution right | Build | Plan | +| ------------ | ----------------------------------------------------- | ----- | ---- | +| proof-only | a rule that only Proof states | 6 | 3 | +| code-visible | a rule Proof states and the code also shows | 5 | 1 | +| stale | the code, where Proof is out of date | 4 | 2 | +| control | nothing about Proof; measures the cost of carrying it | 3 | 2 | + +Every session gets the same code and the same request. The arms differ in +how Proof reaches the agent: + +| Arm | Proof in the session | +| ---------- | -------------------------------------------------------------- | +| `none` | none: no records, no trailers, no CLI, no skill | +| `skill` | available through the proof skill and `./flatbread proof` | +| `find` | as `skill`, plus "run `proof find` on the request first" | +| `findcode` | as `find`, plus `find --path ` before edits and `--refs` | +| `brief` | as `skill`, plus `proof brief` of every Effort pasted in | +| `dump` | as `skill`, plus every live record pasted in | +| `dump10x` | `dump` on the 10x graph (the burden ceiling, one run) | + +A build session is all-green when its acceptance, constraint and +regression tests all pass. The primary endpoints are all-green on the +proof-only tasks (does Proof help?) and on the stale tasks (does stale +Proof hurt?), each arm against `none`, with Holm correction. Cost (first +turn, context and processed tokens, turns, wall time) answers the burden +question. Leakage flags drop sessions that read outside their clone or +reach the hidden tests. + +## Build the beds + +Beds and hidden tests live outside the repository, in `` (the key +maps hold the answers). Builds are deterministic: a rebuild gives the same +hashes. + +```sh +node scripts/gen-bed.mjs --out /bed +node scripts/gen-bed.mjs --variant none --out /bed-none +node scripts/gen-bed.mjs --scale 10 --out /bed-x10 +node scripts/check-bed.mjs --proof /bed --none /bed-none --x10 /bed-x10 +``` + +`--check-each` runs the bed's suite at every commit. + +## Offline gate (free) + +```sh +node scripts/gate.mjs --hidden /hidden --none /bed-none \ + --bed /bed --x10 /bed-x10 +``` + +It assembles `/tasks/`, runs the static checks (no feature word in +Proof, no Proof-only rule anywhere in the none bed, prompts name no file or +symbol), and grades each build task's base tree, reference patch and naive +patch: base fails acceptance, reference passes everything, naive fails the +group it is meant to fail. It exits 1 on any failure. + +## Run + +Prepare first and read what a session would see (no model is called): + +```sh +node scripts/run.mjs /runs --name r1 --bed /bed --prepare-only --ids B-P1 +``` + +Then run for real (`--arms`, `--ids`, `--only `, +`--jobs`, `--model`): + +```sh +node scripts/run.mjs /runs --name r1 --bed /bed --model claude-haiku-5-5 --jobs 8 +``` + +Each session gets a fresh clone and the whole task text on stdin. A rerun +skips finished sessions and retries transient API errors. + +## Grade, judge, compare + +```sh +node scripts/grade.mjs /runs --name r1 --tasks /tasks \ + --none /bed-none --bed /bed [--bed10x /bed-x10] +node scripts/judge.mjs /runs --name r1 --tasks /tasks [--dry-run] +node scripts/compare.mjs /runs --runs r1,r2,r3 --out /runs/compare.json +``` + +`grade.mjs` applies each session's diff to the none bed, restores the +bed's own tests, adds the hidden tests and runs them without network. +`judge.mjs` strips Proof ids and wording from each PLAN.md so the judge +cannot tell the arm, then scores it against the rubric twice; +`--dry-run` prints the blinded prompt. `compare.mjs` pairs each arm with +`none` by task and run. + +Before the full run, the rubric gate checks that the rubrics separate the +hand-written good, trap and plausible plans (it calls the judge): + +```sh +node scripts/judge.mjs --gate --tasks /tasks --out /gate-judge.json +``` + +## Pilot and full run + +The pilot is one build task per category in the six primary arms, once, +on Haiku: 24 sessions, about $1.40 (range $1 to $4), plus about $3 for +the rubric gate. With `P4` set to the `` folder: + +```sh +node scripts/run.mjs $P4/runs --name pilot --bed $P4/bed \ + --ids B-P1,B-S3,B-V4,B-K1 --model claude-haiku-5-5 --jobs 8 +node scripts/grade.mjs $P4/runs --name pilot --tasks $P4/tasks \ + --none $P4/bed-none --bed $P4/bed +node scripts/compare.mjs $P4/runs --runs pilot --out $P4/runs/pilot.json +node scripts/judge.mjs --gate --tasks $P4/tasks --out $P4/gate-judge.json +``` + +The full run is 420 primary sessions (build tasks 3 runs, plan tasks 2) +and 26 `dump10x` sessions, about $65 to $85 and 7.5 hours at 8 jobs: + +```sh +for r in r1 r2; do + node scripts/run.mjs $P4/runs --name $r --bed $P4/bed --model claude-haiku-5-5 --jobs 8 +done +node scripts/run.mjs $P4/runs --name r3 --bed $P4/bed --only build --model claude-haiku-5-5 --jobs 8 +node scripts/run.mjs $P4/runs --name r1 --bed $P4/bed --arms dump10x --model claude-haiku-5-5 --jobs 4 +for r in r1 r2 r3; do + node scripts/grade.mjs $P4/runs --name $r --tasks $P4/tasks \ + --none $P4/bed-none --bed $P4/bed --bed10x $P4/bed-x10 + node scripts/judge.mjs $P4/runs --name $r --tasks $P4/tasks +done +node scripts/compare.mjs $P4/runs --runs r1,r2,r3 --out $P4/runs/compare.json +``` + +## Layout + +| Path | What it holds | +| ----------------------- | ----------------------------------------------------------------------------- | +| `spec/app/` | the bed's code at the tip: sources, tests, fixtures, docs | +| `spec/history.mjs` | the 84 commits and the change each one makes | +| `spec/versions/` | whole earlier files for the two changes too large for text pairs | +| `spec/proof.mjs` | the 7 Efforts and every Proof mutation, by commit | +| `spec/filler.mjs` | the 9 filler Efforts of the 10x graph | +| `spec/banned.mjs` | words no record or commit may hold, so Proof never names a task | +| `tasks//` | `task.json`; plan tasks also `rubric.json` and `plans/` | +| `scripts/gen-bed.mjs` | builds a bed | +| `scripts/check-bed.mjs` | checks built beds against each other and the design | +| `scripts/gate.mjs` | the offline gate | +| `scripts/run.mjs` | runs or prepares sessions (`arms.mjs` arm texts, `inject.mjs` brief and dump) | +| `scripts/grade.mjs` | grades sessions, or one patch | +| `scripts/judge.mjs` | blinds and judges plans; the rubric gate | +| `scripts/compare.mjs` | compares each arm with `none` | + +Hidden tests and the reference and naive patches are in +`/hidden//`, never in this repository. `FLATBREAD_PROOF_SERVER` +must stay unset; Proof reads run locally against each clone's own graph. diff --git a/benchmarks/proof-build/scripts/arms.mjs b/benchmarks/proof-build/scripts/arms.mjs new file mode 100644 index 00000000..6874f9b4 --- /dev/null +++ b/benchmarks/proof-build/scripts/arms.mjs @@ -0,0 +1,108 @@ +// The task text one build-benchmark session receives, for each arm. The +// part that is not about Proof is the same in every arm: the folder rule, +// the request, and the closing line. Arms differ only in the block between +// the folder rule and the request, and in what their clone holds (see +// CLONE). Nothing here names a task, a rule, or a record. + +export const ARMS = [ + 'none', + 'skill', + 'brief', + 'find', + 'findcode', + 'dump', + 'dump10x', +]; + +/** + * What each arm's clone is made from and holds. `bed` is the bed variant + * (`none`, `proof`, or `proof10x`); `proof` adds the skill, the + * `./flatbread` wrapper, the node_modules link, and the AGENTS.md line; + * `inject` names the text pasted into the task (`brief` or `dump`). + */ +export const CLONE = { + none: { bed: 'none', proof: false }, + skill: { bed: 'proof', proof: true }, + brief: { bed: 'proof', proof: true, inject: 'brief' }, + find: { bed: 'proof', proof: true }, + findcode: { bed: 'proof', proof: true }, + dump: { bed: 'proof', proof: true, inject: 'dump' }, + dump10x: { bed: 'proof10x', proof: true, inject: 'dump' }, +}; + +/** The line every Proof arm's AGENTS.md gains. `none` has no such line. */ +export const AGENTS_LINE = + 'Project memory (decisions, constraints, findings) lives in Proof; read it with `./flatbread proof …` (see the proof skill).'; + +/** Every arm's CLAUDE.md, so AGENTS.md loads the same way in each. */ +export const CLAUDE_MD = '@AGENTS.md\n'; + +const FIND = + 'Before planning, run `./flatbread proof find --all`, then `./flatbread proof get ` for any hit that bears on the request.'; +const FIND_CODE = + 'Before editing a file, run `./flatbread proof find --path `. Before relying on a Decision, run `./flatbread proof find --refs all`.'; + +/** The arm's block. `inject` is the brief or dump text for those arms. */ +function armBlock(arm, inject) { + switch (arm) { + case 'none': + case 'skill': + return ''; + case 'brief': + return `## Project memory briefs + +The output of \`./flatbread proof brief \` for each Effort in this repository. + +${inject} +`; + case 'find': + return `## Before you start + +${FIND} +`; + case 'findcode': + return `## Before you start + +${FIND} +${FIND_CODE} +`; + case 'dump': + case 'dump10x': + return `## Project memory records + +Every live record in this repository's project memory, grouped by Effort. + +${inject} +`; + default: + throw new Error(`Unknown arm ${arm}`); + } +} + +export const CLOSING = { + build: 'Implement it, add tests, and run the test suite.', + plan: 'Write PLAN.md in the repository root; do not change code.', +}; + +/** + * Markdown for one session: `kind` is `build` or `plan`, `prompt` the + * request in the user's words, `folder` the clone's absolute path, and + * `inject` the pasted text for `brief` and `dump` arms. + */ +export function armText({ arm, kind, prompt, folder, inject = '' }) { + if (!ARMS.includes(arm)) throw new Error(`Unknown arm ${arm}`); + if (!CLOSING[kind]) throw new Error(`Unknown task kind ${kind}`); + if (CLONE[arm].inject && !inject.trim()) + throw new Error(`Arm ${arm} needs its ${CLONE[arm].inject} text`); + const block = armBlock(arm, inject.trim()); + return `# Task + +You are working in a git repository at \`${folder}\`. Work only inside this folder: do not read, list, or search anything outside it. +${block ? `\n${block}` : ''} +## Request + +${prompt.trim()} + +${CLOSING[kind]} +`; +} diff --git a/benchmarks/proof-build/scripts/check-bed.mjs b/benchmarks/proof-build/scripts/check-bed.mjs new file mode 100644 index 00000000..93af17d0 --- /dev/null +++ b/benchmarks/proof-build/scripts/check-bed.mjs @@ -0,0 +1,263 @@ +#!/usr/bin/env node +// Checks built test beds against each other and against the design, for +// free, before any paid session: +// +// node scripts/check-bed.mjs --proof --none [--x10 ] +// +// - every commit has the same code tree in every variant (`git ls-tree` +// with .flatbread-proof/ and flatbread.config.js left out), and the same +// subject, author and date; +// - the none variant has no graph, no config and no `Proof:` trailer in any +// commit, and no rule wording (spec/banned.mjs BANNED_IN_CODE) anywhere in +// its history, code or messages; +// - the proof variants have trailers on the commits spec/proof.mjs says and +// on no others, and the same record ids for the real Efforts; +// - each drift commit carries no trailer and touches no graph file; +// - the x10 graph holds the real graph plus filler Efforts only. +// +// It prints one line per check and exits non-zero on the first failure. +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { COMMITS } from '../spec/history.mjs'; +import { PROOF, DRIFTS, EFFORTS } from '../spec/proof.mjs'; +import { BANNED_IN_CODE } from '../spec/banned.mjs'; +import { FILLER_KEYS } from '../spec/filler.mjs'; + +const PROOF_DIR = '.flatbread-proof'; +const CONFIG = 'flatbread.config.js'; + +function args(argv) { + const out = {}; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i].replace(/^--/, ''); + if (!['proof', 'none', 'x10'].includes(flag)) + throw new Error(`Unknown argument ${argv[i]}`); + out[flag] = resolve(argv[++i]); + } + if (!out.proof || !out.none) + throw new Error( + 'usage: check-bed.mjs --proof --none [--x10 ]' + ); + return out; +} + +const git = (dir, argv) => + execFileSync('git', argv, { + cwd: dir, + encoding: 'utf8', + maxBuffer: 1 << 30, + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + }, + }).trim(); + +let failed = false; +function check(label, ok, detail = '') { + console.log( + `${ok ? 'ok ' : 'FAIL'} ${label}${detail ? ` (${detail})` : ''}` + ); + if (!ok) failed = true; +} + +function log(dir) { + const shas = git(dir, ['rev-list', '--reverse', 'main']).split('\n'); + return shas.map((sha) => ({ + sha, + subject: git(dir, ['log', '-1', '--format=%s', sha]), + who: git(dir, ['log', '-1', '--format=%an <%ae> %aI | %cn <%ce> %cI', sha]), + trailers: git(dir, [ + 'log', + '-1', + '--format=%(trailers:key=Proof,valueonly,separator=%x2C)', + sha, + ]) + .split(',') + .map((s) => s.trim()) + .filter(Boolean), + code: git(dir, ['ls-tree', '-r', sha]) + .split('\n') + .filter( + (l) => !l.includes(`\t${PROOF_DIR}/`) && !l.endsWith(`\t${CONFIG}`) + ) + .join('\n'), + graph: git(dir, ['ls-tree', '-r', '--name-only', sha]) + .split('\n') + .filter((p) => p.startsWith(`${PROOF_DIR}/`)), + })); +} + +function keymap(dir) { + return JSON.parse(readFileSync(`${dir}.keymap.json`, 'utf8')); +} + +function main() { + const o = args(process.argv.slice(2)); + const beds = { + proof: o.proof, + none: o.none, + ...(o.x10 ? { x10: o.x10 } : {}), + }; + const logs = Object.fromEntries( + Object.entries(beds).map(([k, d]) => [k, log(d)]) + ); + const maps = Object.fromEntries( + Object.entries(beds).map(([k, d]) => [k, keymap(d)]) + ); + for (const [name, l] of Object.entries(logs)) + check( + `${name}: ${COMMITS.length} commits on main`, + l.length === COMMITS.length, + `${l.length}` + ); + + // Same code tree, subject, author and date, commit for commit. + for (const other of Object.keys(beds).filter((k) => k !== 'none')) { + const diff = COMMITS.map((c, i) => i).filter( + (i) => logs.none[i].code !== logs[other][i].code + ); + check( + `${other} and none have identical code trees at every commit`, + diff.length === 0, + diff.length + ? `differ at ${diff.map((i) => COMMITS[i].key)}` + : `${COMMITS.length} trees` + ); + const meta = COMMITS.map((c, i) => i).filter( + (i) => + logs.none[i].subject !== logs[other][i].subject || + logs.none[i].who !== logs[other][i].who + ); + check( + `${other} and none have the same subjects, authors and dates`, + meta.length === 0 + ); + } + check( + 'the tips of all variants have the same code tree', + new Set(Object.values(logs).map((l) => l.at(-1).code)).size === 1 + ); + + // The none variant has no Proof at all. + const none = logs.none; + check( + 'none: no Proof: trailer in any commit', + none.every((c) => !c.trailers.length) + ); + check( + 'none: no .flatbread-proof/ or flatbread.config.js in any commit', + none.every((c) => !c.graph.length) && + !git(o.none, ['log', '--all', '--format=%H', '--', CONFIG]) + ); + check('none: no record ids in its key map', !maps.none.records); + const history = git(o.none, [ + 'log', + '-p', + '--all', + '--format=%B', + ]).toLowerCase(); + const leaks = BANNED_IN_CODE.filter((p) => history.includes(p.toLowerCase())); + check( + 'none: no proof-only rule wording anywhere in its history (code, tests, docs, messages)', + leaks.length === 0, + leaks.join(', ') + ); + check( + 'none: no mention of Proof or flatbread in its history', + !/flatbread|proof:/i.test(history) + ); + + // Trailers where the spec puts them, and nowhere else. + for (const name of Object.keys(beds).filter((k) => k !== 'none')) { + const l = logs[name]; + const ids = maps[name].records; + const wrong = COMMITS.flatMap((c, i) => { + const want = (PROOF[c.key]?.trailers ?? []).map((k) => ids[k]).sort(); + const have = [...l[i].trailers].sort(); + return want.join() === have.join() ? [] : [c.key]; + }); + check( + `${name}: trailers match spec/proof.mjs`, + wrong.length === 0, + wrong.length + ? wrong.join(', ') + : `${l.filter((c) => c.trailers.length).length} commits with trailers` + ); + for (const [key, d] of Object.entries(DRIFTS)) { + const i = COMMITS.findIndex((c) => c.key === d.commit); + const touched = git(beds[name], [ + 'show', + '--name-only', + '--format=', + l[i].sha, + ]).split('\n'); + check( + `${name}: ${key} (${d.commit} "${COMMITS[i].subject}") has no trailer and leaves the graph alone`, + !l[i].trailers.length && + !touched.some((p) => p.startsWith(`${PROOF_DIR}/`)) + ); + } + const after = COMMITS.findIndex((c) => c.key === 'c76'); + const graphTouched = l.slice(after).filter((c, j) => { + const touched = git(beds[name], [ + 'show', + '--name-only', + '--format=', + c.sha, + ]).split('\n'); + return ( + touched.some((p) => p.startsWith(`${PROOF_DIR}/`)) && name === 'proof' + ); + }); + if (name === 'proof') + check( + 'proof: no commit from the first drift on touches the graph', + graphTouched.length === 0, + graphTouched.map((c) => c.subject).join('; ') + ); + } + + // The same record ids for the real Efforts in proof and x10. + if (maps.x10) { + const real = Object.keys(EFFORTS); + const p = maps.proof.records; + const x = maps.x10.records; + const differ = Object.keys(p).filter((k) => p[k] !== x[k]); + check( + 'x10 has every real record with the same id', + differ.length === 0, + differ.join(', ') + ); + const extra = Object.keys(x).filter((k) => !(k in p)); + check( + 'x10 adds only filler Efforts', + extra.every((k) => FILLER_KEYS.includes(k.split('.')[0])), + `${extra.length} filler keys` + ); + check( + 'x10 filler is outside the real Efforts', + extra.every((k) => !real.includes(k.split('.')[0])) + ); + const files = logs.x10.at(-1).graph.filter((f) => f.endsWith('.md')).length; + const proofFiles = logs.proof + .at(-1) + .graph.filter((f) => f.endsWith('.md')).length; + check( + 'x10 graph is about ten times the real one', + files >= 8 * proofFiles, + `${files} vs ${proofFiles} record files` + ); + } + for (const [name, dir] of Object.entries(beds)) + check( + `${name}: working tree clean, main checked out`, + !git(dir, ['status', '--porcelain']) && + git(dir, ['rev-parse', '--abbrev-ref', 'HEAD']) === 'main' && + existsSync(dir) + ); + if (failed) process.exitCode = 1; +} + +main(); diff --git a/benchmarks/proof-build/scripts/common.mjs b/benchmarks/proof-build/scripts/common.mjs new file mode 100644 index 00000000..7dfe194d --- /dev/null +++ b/benchmarks/proof-build/scripts/common.mjs @@ -0,0 +1,574 @@ +// What run.mjs, grade.mjs, judge.mjs and compare.mjs share: where a session +// keeps its files, how tasks load, git, and the transcript scans that flag +// leaks and count Proof use. +import { execFileSync, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { dirname, join, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const BENCH = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +export const ROOT = resolve(BENCH, '..', '..'); +export const FLATBREAD_BIN = join( + ROOT, + 'packages', + 'flatbread', + 'bin', + 'flatbread.js' +); + +/** Files a session's clone gains from the runner, kept out of git status. */ +export const SESSION_FILES = [ + '/flatbread', + '/node_modules', + '/CLAUDE.md', + '/.claude/', + '/.flatbread/', +]; + +/** + * Paths left out of `session.diff`: Proof's own files, the runner's files, + * and PLAN.md, which is captured on its own. + */ +export const DIFF_EXCLUDE = [ + '.flatbread-proof', + '.flatbread', + '.claude', + 'AGENTS.md', + 'CLAUDE.md', + 'flatbread.config.js', + 'flatbread', + 'node_modules', + 'PLAN.md', +]; + +/** Files in a task folder that never reach a session or a grading tree. */ +const TASK_PRIVATE = /^(task\.json|rubric\.json|.*\.patch|plans)$/; + +/** A test file, as the bed's own suite names them. */ +export const TEST_FILE = /\.test\.(ts|mts|mjs|js)$/; + +export function git(cwd, ...argv) { + return execFileSync('git', argv, { + cwd, + encoding: 'utf8', + maxBuffer: 256 * 1024 * 1024, + stdio: ['ignore', 'pipe', 'pipe'], + env: { + ...process.env, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_AUTHOR_NAME: 'bench', + GIT_AUTHOR_EMAIL: 'bench@example.invalid', + GIT_COMMITTER_NAME: 'bench', + GIT_COMMITTER_EMAIL: 'bench@example.invalid', + }, + }); +} + +export const sha256 = (text) => createHash('sha256').update(text).digest('hex'); + +/** tune or holdout, by the first byte of sha256(id), as in proof-recall. */ +export function splitOf(id) { + return createHash('sha256').update(id).digest()[0] % 2 === 0 + ? 'tune' + : 'holdout'; +} + +export const isUnder = (path, dir) => + path === dir || path.startsWith(dir.endsWith(sep) ? dir : dir + sep); + +/** Every task under `dir`: one folder per task holding task.json. */ +export function loadTasks(dir) { + const tasks = []; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory() || entry.name.startsWith('_')) continue; + const file = join(dir, entry.name, 'task.json'); + if (!existsSync(file)) continue; + const task = JSON.parse(readFileSync(file, 'utf8')); + if (task.id !== entry.name) + throw new Error(`${file}: id ${task.id} differs from its folder`); + if (!['build', 'plan'].includes(task.kind)) + throw new Error(`${file}: kind must be build or plan`); + if (!task.prompt?.trim()) throw new Error(`${file}: no prompt`); + tasks.push({ ...task, dir: join(dir, entry.name) }); + } + return tasks.sort((a, b) => a.id.localeCompare(b.id)); +} + +/** + * Tasks picked by `--only` (kinds or categories) and `--ids`. Both empty + * means every task. + */ +export function pickTasks(tasks, { only = [], ids = [] }) { + const known = new Set(tasks.map((t) => t.id)); + for (const id of ids) if (!known.has(id)) throw new Error(`No task ${id}`); + return tasks.filter( + (t) => + (!ids.length || ids.includes(t.id)) && + (!only.length || only.includes(t.kind) || only.includes(t.category)) + ); +} + +/** Hidden files a task copies into `test/_hidden/` at grading time. */ +export function hiddenFiles(taskDir) { + return readdirSync(taskDir).filter((name) => !TASK_PRIVATE.test(name)); +} + +/** Where one session keeps its files, outside its clone. */ +export function sessionDir(work, name, arm, id) { + return join(work, name, arm, id); +} + +/** The clone's path, which names neither the arm nor the task. */ +export function cloneDir(work, name, arm, id) { + return join( + work, + name, + 'clones', + sha256(`${name}\0${arm}\0${id}`).slice(0, 12) + ); +} + +export async function readJson(path, fallback = null) { + try { + return JSON.parse(await readFile(path, 'utf8')); + } catch { + return fallback; + } +} + +/** Every session folder under `/`: [{arm, id, dir}]. */ +export function listSessions(work, name) { + const base = join(work, name); + const out = []; + if (!existsSync(base)) return out; + for (const arm of readdirSync(base, { withFileTypes: true })) { + if (!arm.isDirectory() || arm.name === 'clones') continue; + for (const id of readdirSync(join(base, arm.name), { + withFileTypes: true, + })) + if (id.isDirectory()) + out.push({ + arm: arm.name, + id: id.name, + dir: join(base, arm.name, id.name), + }); + } + return out; +} + +/** + * Runs a command with a hard timeout, killing its whole process group. + * Resolves with { code, timedOut, stdout, stderr, ms }. + */ +export function runCommand(cmd, argv, { cwd, env, timeout, input } = {}) { + return new Promise((done) => { + const start = Date.now(); + const child = spawn(cmd, argv, { + cwd, + env, + detached: true, + stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + let timedOut = false; + child.stdout.on('data', (c) => (stdout += c)); + child.stderr.on('data', (c) => (stderr += c)); + if (input !== undefined) child.stdin.end(input); + const kill = (signal) => { + try { + process.kill(-child.pid, signal); + } catch { + // Already gone. + } + }; + const timer = + timeout && + setTimeout(() => { + timedOut = true; + kill('SIGTERM'); + setTimeout(() => kill('SIGKILL'), 5000).unref(); + }, timeout); + child.on('close', (code) => { + clearTimeout(timer); + done({ code, timedOut, stdout, stderr, ms: Date.now() - start }); + }); + }); +} + +/** + * Tool calls in a stream-json transcript: [{name, input, result}], where + * `result` is the text of the matching tool_result, when there is one. + */ +export async function toolCalls(transcriptPath) { + let text = ''; + try { + text = await readFile(transcriptPath, 'utf8'); + } catch { + return []; + } + const calls = []; + const byId = new Map(); + for (const line of text.split('\n')) { + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + for (const part of entry.message?.content ?? []) { + if ( + entry.type === 'assistant' && + part.type === 'tool_use' && + !byId.has(part.id) + ) { + const call = { name: part.name, input: part.input ?? {} }; + byId.set(part.id, call); + calls.push(call); + } + if (entry.type === 'user' && part.type === 'tool_result') { + const call = byId.get(part.tool_use_id); + if (!call) continue; + const content = Array.isArray(part.content) + ? part.content + .map((c) => (typeof c === 'string' ? c : c.text ?? '')) + .join('\n') + : String(part.content ?? ''); + call.result = (call.result ?? '') + content; + } + } + } + return calls; +} + +/** The `init` and `result` events of a transcript. */ +export async function transcriptEnds(transcriptPath) { + const out = { init: null, result: null }; + let text = ''; + try { + text = await readFile(transcriptPath, 'utf8'); + } catch { + return out; + } + for (const line of text.split('\n')) { + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + if (entry.type === 'system' && entry.subtype === 'init') out.init = entry; + if (entry.type === 'result') out.result = entry; + } + return out; +} + +const READ_COMMANDS = new Set([ + 'list', + 'find', + 'get', + 'records', + 'brief', + 'relations', + 'blocking-decisions', + 'state', + 'help', +]); + +// Absolute paths in shell text: a slash after a start, space, quote, `=`, +// `(` or `:`, then at least one path character, so a lone `/` (division) +// and `//` or `/*` (comments) are not paths. +const ABS_PATH = /(?:^|[\s'"=(:<>])(\/[\w.@+%~,-]+(?:\/[\w.@+%~,*-]*)*)/g; +// Relative paths that start with `..`. +const DOTDOT = + /(?:^|[\s'"=(:<>])(\.\.(?:\/[\w.@+%~,*-]*)*)(?=$|[\s'"):;|&<>])/g; +// Commands whose bare `/` argument means the filesystem root. +const FS_COMMANDS = + /^\s*(?:sudo\s+)?(ls|find|du|tree|grep|rg|ag|cat|stat|locate|fd|head|tail|less|file|cp|mv|rm|tar)\b/; +// The script text of `node -e`, `python -c`, `sh -c` and the like. +const INLINE_SCRIPT = + /(\s(?:-e|-c|--eval|--print)\s+)('(?:[^']*)'|"(?:\\.|[^"\\])*")/g; + +/** + * Splits a Bash command into shell text and body text. Body text is what a + * program reads rather than what the shell resolves: heredoc bodies and + * inline scripts (`node -e "…"`). Only shell text is checked for `..` and + * relative paths; body text is still checked for absolute paths under the + * protected roots, since a heredoc can feed a shell. + */ +export function splitShell(command) { + const shell = []; + const body = []; + const lines = command.split('\n'); + for (let i = 0; i < lines.length; i++) { + let line = lines[i]; + line = line.replace(INLINE_SCRIPT, (_, flag, script) => { + body.push(script); + return `${flag}''`; + }); + shell.push(line); + const heredoc = line.match(/<<-?\s*(['"]?)([\w.-]+)\1/); + if (!heredoc) continue; + const end = heredoc[2]; + const strip = line.includes('<<-'); + for (i++; i < lines.length; i++) { + const text = strip ? lines[i].replace(/^\t+/, '') : lines[i]; + if (text === end) break; + body.push(lines[i]); + } + } + return { shell: shell.join('\n'), body: body.join('\n') }; +} + +const PROOF_CLI = /flatbread\s+proof\b/; + +/** + * Scans a session's tool calls for leaks and Proof use. + * + * - `left_folder`: a file tool's path, a Glob or Grep path or absolute + * pattern, or a path in a Bash command lies outside `clone`. In Bash, a + * relative path starts at the shell's folder, which follows `cd` inside + * the clone; a `cd` out of it counts as leaving. Absolute paths outside the + * clone count only when they are the clone's ancestors or fall under a + * `protected` root (the work, task, grade, hidden and bed folders, the + * p4 folder and this repository), so `/dev/null` or `/usr/bin/env` do + * not. Heredoc bodies and inline scripts are checked only for absolute + * paths under a protected root, so code such as `import '../src/x.ts'`, + * `100 / 3` or a `// comment` written through the shell is not a leak. + * - `hidden_read`: a tool input names a hidden root (the task, hidden and + * grading folders, the gate log), `test/_hidden`, or a task's reference or + * naive patch or rubric. + * - `proof_reads` / `proof_writes`: `flatbread proof ` calls, + * split by whether the command only reads; direct edits under + * `.flatbread-proof/` count as writes. + * - `proof_files_read`: Read calls on files under `.flatbread-proof/`. + * - `proof_search_exposure`: searches and other calls that are not Proof + * commands but whose result showed text from `.flatbread-proof/` (a + * repository-wide Grep, `grep -r`, `find`, `ls -R`). + * - `proof_touch`: any mention of `.flatbread-proof` or `flatbread proof`, + * which a `none` session should never make. + * - `opened_cited`: Read calls on a path a code Citation names. + */ +export function scanCalls( + calls, + { clone, protectedRoots = [], hiddenRoots = [], citedPaths = [] } +) { + const out = { + left_folder: false, + left_folder_evidence: [], + hidden_read: false, + hidden_read_evidence: [], + proof_reads: 0, + proof_writes: 0, + proof_files_read: 0, + proof_search_exposure: 0, + proof_touch: false, + skill_calls: 0, + opened_cited: 0, + }; + const cited = new Set(citedPaths); + const isProtected = (abs) => + isUnder(clone, abs) || protectedRoots.some((root) => isUnder(abs, root)); + // `cwd` is where a relative path starts: the clone for file tools, and + // the shell's folder for Bash, which follows `cd` within and across calls + // (as Claude Code's Bash keeps its folder while it stays in the project). + const outside = (path, fromShell, cwd = clone) => { + const abs = resolve(cwd, path); + if (isUnder(abs, clone)) return false; + if (!fromShell || !path.startsWith('/')) return true; + return isProtected(abs); + }; + let shellCwd = clone; + const flag = (path, call) => { + out.left_folder = true; + if (out.left_folder_evidence.length < 5) + out.left_folder_evidence.push(`${call.name}: ${path}`); + }; + for (const call of calls) { + const input = call.input ?? {}; + const text = JSON.stringify(input); + if (call.name === 'Skill') out.skill_calls++; + if ( + hiddenRoots.some((root) => text.includes(root)) || + /test\/_hidden|(reference|naive)[\w-]*\.patch|rubric\.json|gate\.(log|json)/.test( + text + ) + ) { + out.hidden_read = true; + if (out.hidden_read_evidence.length < 5) + out.hidden_read_evidence.push(`${call.name}: ${text.slice(0, 160)}`); + } + if (/\.flatbread-proof|flatbread\s+proof/.test(text)) + out.proof_touch = true; + const isProofCommand = + call.name === 'Bash' && PROOF_CLI.test(input.command ?? ''); + const readsProofFile = + ['Read', 'Write', 'Edit', 'NotebookEdit'].includes(call.name) && + /\.flatbread-proof\//.test(input.file_path ?? ''); + if ( + !isProofCommand && + !readsProofFile && + typeof call.result === 'string' && + /\.flatbread-proof\//.test(call.result) + ) + out.proof_search_exposure++; + const paths = [input.file_path, input.path, input.notebook_path].filter( + (p) => typeof p === 'string' && p + ); + if ( + ['Glob', 'Grep'].includes(call.name) && + typeof input.pattern === 'string' && + input.pattern.startsWith('/') + ) + paths.push(input.pattern); + for (const path of paths) if (outside(path, false)) flag(path, call); + if (call.name === 'Read' && typeof input.file_path === 'string') { + const rel = resolve(clone, input.file_path).slice(clone.length + 1); + if (cited.has(rel)) out.opened_cited++; + if (readsProofFile) out.proof_files_read++; + } + if ( + ['Write', 'Edit', 'NotebookEdit'].includes(call.name) && + /\.flatbread-proof\//.test(input.file_path ?? '') + ) + out.proof_writes++; + if (call.name === 'Bash' && typeof input.command === 'string') { + const command = input.command; + const { shell, body } = splitShell(command); + let cwd = shellCwd; + for (const segment of shell.split(/&&|\|\||;|\||\n/)) { + const cd = segment.match(/^\s*cd(?:\s+(\S+))?\s*$/); + if (cd) { + const arg = (cd[1] ?? '~').replace(/^['"]|['"]$/g, ''); + if (arg === '-') continue; + const target = /^[~$]/.test(arg) ? '/' : resolve(cwd, arg); + if (isUnder(target, clone)) cwd = target; + else { + flag(`cd ${arg}`, call); + cwd = clone; + } + continue; + } + if (FS_COMMANDS.test(segment) && /\s\/(?=\s|$)/.test(segment)) + flag('/', call); + for (const m of segment.matchAll(ABS_PATH)) + if (outside(m[1], true, cwd)) flag(m[1], call); + for (const m of segment.matchAll(DOTDOT)) + if (outside(m[1], true, cwd)) flag(m[1], call); + } + for (const m of body.matchAll(ABS_PATH)) { + const abs = resolve(m[1]); + if (!isUnder(abs, clone) && protectedRoots.some((r) => isUnder(abs, r))) + flag(m[1], call); + } + shellCwd = cwd; + for (const m of command.matchAll(/flatbread\s+proof\s+([a-z][\w-]*)/g)) + if (READ_COMMANDS.has(m[1])) out.proof_reads++; + else out.proof_writes++; + } + } + return out; +} + +/** Code paths that the graph's code Citations name (`path#symbol@sha`). */ +export function citedPaths(proofRoot) { + const dir = join(proofRoot, 'citations'); + if (!existsSync(dir)) return []; + const paths = new Set(); + for (const file of readdirSync(dir).filter((f) => f.endsWith('.md'))) { + const text = readFileSync(join(dir, file), 'utf8'); + const body = text.replace(/^---\n[\s\S]*?\n---\n/, ''); + for (const m of body.matchAll(/^([\w./-]+\.\w+)#[\w$.]+(?:@\w+)?/gm)) + paths.add(m[1]); + } + return [...paths]; +} + +/** Environment variables that tie a process to the session that started it. */ +export const SESSION_VARS = [ + 'CLAUDE_CODE_SESSION_ID', + 'CLAUDE_CODE_REMOTE_SESSION_ID', + 'CLAUDE_CODE_REMOTE_SDK_URL', + 'CLAUDE_CODE_MESSAGING_SOCKET', + 'CLAUDE_CODE_MESSAGING_TOKEN', + 'CLAUDE_SESSION_INGRESS_TOKEN_FILE', + 'CLAUDE_CODE_CHILD_SESSION', + 'CLAUDE_CODE_SESSION_ATTENDED', + 'CLAUDE_CODE_SYNC_SESSION_REFS', + 'CLAUDE_CODE_TEE_SDK_STDOUT', + 'CLAUDE_PID', + 'CLAUDECODE', + 'CLAUDE_CODE_ENTRYPOINT', + 'FLATBREAD_PROOF_SERVER', +]; + +export function cleanEnv() { + const env = { ...process.env }; + for (const key of SESSION_VARS) delete env[key]; + return env; +} + +/** Parses `--flag value` and bare arguments against a spec of defaults. */ +export function parseArgs(argv, spec, { lists = [], flags = [] } = {}) { + const out = { ...spec, _: [] }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith('--')) { + out._.push(arg); + continue; + } + const key = arg.slice(2).replace(/-(\w)/g, (_, c) => c.toUpperCase()); + if (flags.includes(key)) { + out[key] = true; + continue; + } + if (!(key in spec)) throw new Error(`Unknown argument ${arg}`); + const value = argv[++i]; + if (value === undefined) throw new Error(`${arg} needs a value`); + out[key] = lists.includes(key) + ? value.split(',').filter(Boolean) + : typeof spec[key] === 'number' + ? Number(value) + : value; + } + return out; +} + +let netnsWorks = null; +/** + * `[cmd, argv]` that runs `cmd argv` with no network when this machine + * allows it (`unshare -n`, which needs root or user namespaces), and as + * it is otherwise. `isolated` says which. + */ +export function withoutNetwork(cmd, argv) { + if (netnsWorks === null) { + try { + execFileSync('unshare', ['-n', 'true'], { stdio: 'ignore' }); + netnsWorks = true; + } catch { + try { + execFileSync('unshare', ['-rn', 'true'], { stdio: 'ignore' }); + netnsWorks = '-rn'; + } catch { + netnsWorks = false; + } + } + } + if (!netnsWorks) return { cmd, argv, isolated: false }; + return { + cmd: 'unshare', + argv: [netnsWorks === '-rn' ? '-rn' : '-n', '--', cmd, ...argv], + isolated: true, + }; +} + +/** The environment for grading runs: no session variables, no proxy. */ +export function gradeEnv() { + const env = cleanEnv(); + for (const key of Object.keys(env)) + if (/^(https?|all|no)_proxy$/i.test(key)) delete env[key]; + return env; +} diff --git a/benchmarks/proof-build/scripts/compare.mjs b/benchmarks/proof-build/scripts/compare.mjs new file mode 100644 index 00000000..fc119269 --- /dev/null +++ b/benchmarks/proof-build/scripts/compare.mjs @@ -0,0 +1,459 @@ +#!/usr/bin/env node +// Compares every arm with `none`, paired by task and run. +// +// node scripts/compare.mjs --runs r1,r2 +// [--arms skill,brief,find,findcode,dump,dump10x] [--base none] +// [--split tune|holdout|all] [--only build|plan|] +// [--ids a,b] [--keep-flagged] [--errors drop|fail] [--out ] +// +// It reads each session's session.json, grade.json and judge.json under +// ////. A row is one (task, run) pair that both +// the arm and the base have graded. +// +// Exclusion: when any compared arm's session for a (task, run) is +// `excluded` by grade.mjs (left_folder, hidden_read, or a `none` session +// that found Proof), that (task, run) is dropped from every comparison, as +// DESIGN.md §7 says. --keep-flagged keeps them. Flag counts by arm are in +// `flags`. Timed-out sessions stay in and count as failures. Sessions that +// ended in a non-transient API error (run.mjs `api_error`) are counted in +// `flags` and, by default, their (task, run) is dropped like a flagged one; +// --errors fail keeps them as failures instead. +// +// Build tasks: all-green, acceptance, constraints and regressions, each +// with the arm's and the base's pass counts, the pairs only one side passed +// (`wins`, `losses`), a two-sided exact sign test over those pooled pairs +// (`p`), and the stricter task-level sign test that counts each task once, +// as a win for the side that passed more often over the runs +// (`taskSignTest`). A constraint pass counts only when acceptance passed +// too, because constraint tests skip when the feature is missing and an +// empty diff would otherwise pass them; `constraints_raw` is the plain +// count, for reference. +// +// Primary endpoints (DESIGN.md §7): all-green on proof-only tasks (H1) +// and on stale tasks (H3), each arm against `none`, ten sign tests +// Holm-corrected together (`holm_primary`, keyed `:`). The +// pooled all-green over every build task is secondary, Holm-corrected +// across the five primary arms (`holm_all_green`). `dump10x` is compared +// with `dump` too, without correction. +// +// Plan tasks: the mean rubric score of each side, a paired two-sided +// Wilcoxon signed-rank test (exact, ties averaged), and must-avoid +// violations. +// +// Cost, over build and plan pairs apart: first-turn tokens (the system +// prompt plus the task text, so the pasted brief or dump shows here), +// context tokens (every fresh input and output token, first turn +// included), read tokens (context tokens without the first turn: what the +// session read and wrote by itself), processed tokens (context plus cache +// reads, which counts a pasted block once per turn), tool calls, turns, +// wall time, Proof reads, direct reads of Proof files, and searches whose +// results showed Proof files (`proof_search_exposure`), as medians, with +// the median paired ratio arm/base for the token measures and wall time. +// H4 is read on context and processed tokens, which include what was +// forced in; read tokens alone would hide it. +// +// Everything is also given per category (`byCategory`), which is +// descriptive. --split keeps only the tune or holdout tasks, split by the +// first byte of sha256(task id) as in proof-recall. +import { writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { signTest } from '../../proof-recall/scripts/compare.mjs'; +import { ARMS } from './arms.mjs'; +import { listSessions, parseArgs, readJson, splitOf } from './common.mjs'; + +const PRIMARY = ['skill', 'brief', 'find', 'findcode', 'dump']; +const PRIMARY_CATEGORIES = ['proof-only', 'stale']; +const BUILD_GROUPS = [ + 'all_green', + 'acceptance', + 'constraints', + 'constraints_raw', + 'regressions', +]; +const RATIOS = [ + 'first_turn_tokens', + 'context_tokens', + 'read_tokens', + 'processed_tokens', + 'wall_ms', +]; + +function median(values) { + const v = values.filter((x) => Number.isFinite(x)).sort((a, b) => a - b); + if (!v.length) return null; + const mid = v.length / 2; + return v.length % 2 ? v[Math.floor(mid)] : (v[mid - 1] + v[mid]) / 2; +} + +const round = (x, digits = 0) => + x === null || x === undefined ? null : +x.toFixed(digits); + +/** Two-sided Wilcoxon signed-rank p for paired differences `d`. */ +export function wilcoxon(d) { + const nz = d.filter((x) => x !== 0); + const n = nz.length; + if (!n) return { n: 0, w_plus: 0, p: 1 }; + const order = nz + .map((x, i) => ({ abs: Math.abs(x), i })) + .sort((a, b) => a.abs - b.abs); + const ranks = new Array(n); + for (let i = 0; i < n; ) { + let j = i; + while (j + 1 < n && order[j + 1].abs === order[i].abs) j++; + for (let k = i; k <= j; k++) ranks[order[k].i] = (i + j + 2) / 2; + i = j + 1; + } + const wPlus = nz.reduce((s, x, i) => s + (x > 0 ? ranks[i] : 0), 0); + if (n <= 25) { + // Exact: the null distribution of W+ over all 2^n sign patterns, on + // doubled ranks so that tied (half) ranks stay integers. + const r2 = ranks.map((r) => Math.round(r * 2)); + const max = r2.reduce((a, b) => a + b, 0); + let dist = new Float64Array(max + 1); + dist[0] = 1; + for (const r of r2) { + const next = new Float64Array(max + 1); + for (let s = 0; s <= max; s++) + if (dist[s]) { + next[s] += dist[s] / 2; + next[s + r] += dist[s] / 2; + } + dist = next; + } + const w = Math.round(wPlus * 2); + let low = 0; + let high = 0; + for (let s = 0; s <= max; s++) { + if (s <= w) low += dist[s]; + if (s >= w) high += dist[s]; + } + return { n, w_plus: wPlus, p: Math.min(1, 2 * Math.min(low, high)) }; + } + const mean = (n * (n + 1)) / 4; + const ties = {}; + for (const r of ranks) ties[r] = (ties[r] ?? 0) + 1; + const tie = Object.values(ties).reduce((s, t) => s + (t ** 3 - t), 0); + const sd = Math.sqrt((n * (n + 1) * (2 * n + 1)) / 24 - tie / 48); + const z = (Math.abs(wPlus - mean) - 0.5) / sd; + // Two-sided normal tail, by the Abramowitz-Stegun erfc approximation. + const t = 1 / (1 + 0.3275911 * (z / Math.SQRT2)); + const erfc = + t * + (0.254829592 + + t * + (-0.284496736 + + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429)))) * + Math.exp(-((z / Math.SQRT2) ** 2)); + return { n, w_plus: wPlus, p: Math.min(1, erfc) }; +} + +/** Holm step-down adjustment of `ps` (a map name → p). */ +export function holm(ps) { + const entries = Object.entries(ps).sort((a, b) => a[1] - b[1]); + const out = {}; + let running = 0; + entries.forEach(([name, p], i) => { + running = Math.max(running, Math.min(1, p * (entries.length - i))); + out[name] = +running.toFixed(4); + }); + return out; +} + +function options(argv) { + const o = parseArgs( + argv, + { + runs: [], + arms: [], + base: 'none', + split: 'all', + only: [], + ids: [], + out: null, + errors: 'drop', + }, + { lists: ['runs', 'arms', 'only', 'ids'], flags: ['keepFlagged'] } + ); + if (!['drop', 'fail'].includes(o.errors)) + throw new Error('--errors must be drop or fail'); + if (!o._[0] || !o.runs.length) + throw new Error('usage: compare.mjs --runs r1[,r2] [options]'); + if (!['tune', 'holdout', 'all'].includes(o.split)) + throw new Error('--split must be one of tune, holdout, all'); + o.dir = resolve(o._[0]); + return o; +} + +async function loadRows(o) { + const rows = []; + for (const run of o.runs) + for (const s of listSessions(o.dir, run)) { + const session = await readJson(join(s.dir, 'session.json')); + const grade = await readJson(join(s.dir, 'grade.json')); + if (!session?.done || !grade) continue; + const judge = await readJson(join(s.dir, 'judge.json')); + rows.push({ + key: `${s.id}#${run}`, + task: s.id, + run, + arm: s.arm, + kind: grade.kind, + category: grade.category ?? session.category ?? 'none', + session, + grade, + judge, + }); + } + return rows.filter( + (r) => + (o.split === 'all' || splitOf(r.task) === o.split) && + (!o.ids.length || o.ids.includes(r.task)) && + (!o.only.length || o.only.includes(r.kind) || o.only.includes(r.category)) + ); +} + +const readTokens = (r) => + (r.session.cost?.context_tokens ?? NaN) - (r.session.cost?.fixed_tokens ?? 0); +const metric = { + first_turn_tokens: (r) => r.session.cost?.fixed_tokens ?? NaN, + context_tokens: (r) => r.session.cost?.context_tokens ?? NaN, + read_tokens: readTokens, + processed_tokens: (r) => r.session.cost?.processed_tokens ?? NaN, + tool_calls: (r) => r.session.cost?.tool_calls ?? NaN, + turns: (r) => r.session.cost?.turns ?? NaN, + wall_ms: (r) => r.session.wall_ms ?? NaN, + proof_reads: (r) => r.grade.flags?.proof_reads ?? 0, + proof_files_read: (r) => r.grade.flags?.proof_files_read ?? 0, + proof_search_exposure: (r) => r.grade.flags?.proof_search_exposure ?? 0, +}; +const passed = (r, group) => { + if (group === 'all_green') return r.grade.all_green === true; + if (group === 'constraints') + return ( + r.grade.acceptance?.pass === true && r.grade.constraints?.pass !== false + ); + if (group === 'constraints_raw') return r.grade.constraints?.pass !== false; + return r.grade[group]?.pass !== false; +}; + +function binary(pairs, group) { + const wins = pairs.filter(([a, b]) => passed(a, group) && !passed(b, group)); + const losses = pairs.filter( + ([a, b]) => !passed(a, group) && passed(b, group) + ); + const byTask = new Map(); + for (const [a, b] of pairs) + byTask.set( + a.task, + (byTask.get(a.task) ?? 0) + + Number(passed(a, group)) - + Number(passed(b, group)) + ); + const tw = [...byTask.values()].filter((m) => m > 0).length; + const tl = [...byTask.values()].filter((m) => m < 0).length; + return { + arm: pairs.filter(([a]) => passed(a, group)).length, + base: pairs.filter(([, b]) => passed(b, group)).length, + wins: wins.map(([a]) => a.key), + losses: losses.map(([a]) => a.key), + p: +signTest(wins.length, losses.length).toFixed(4), + taskSignTest: { arm: tw, base: tl, p: +signTest(tw, tl).toFixed(4) }, + }; +} + +function plans(pairs) { + const scored = pairs.filter(([a, b]) => a.judge && b.judge); + const d = scored.map(([a, b]) => a.judge.score - b.judge.score); + const mean = (xs) => + xs.length ? +(xs.reduce((s, x) => s + x, 0) / xs.length).toFixed(3) : null; + return { + pairs: scored.length, + unjudged: pairs.length - scored.length, + score: { + arm: mean(scored.map(([a]) => a.judge.score)), + base: mean(scored.map(([, b]) => b.judge.score)), + }, + wins: d.filter((x) => x > 0).length, + losses: d.filter((x) => x < 0).length, + wilcoxon: (({ n, w_plus, p }) => ({ n, w_plus, p: +p.toFixed(4) }))( + wilcoxon(d) + ), + avoid_violations: { + arm: scored.reduce((n, [a]) => n + a.judge.avoid_violations, 0), + base: scored.reduce((n, [, b]) => n + b.judge.avoid_violations, 0), + }, + plan_changed_code: { + arm: pairs.filter(([a]) => a.grade.plan_changed_code).length, + base: pairs.filter(([, b]) => b.grade.plan_changed_code).length, + }, + }; +} + +function cost(pairs) { + const out = {}; + for (const [name, get] of Object.entries(metric)) { + const a = pairs.map(([x]) => get(x)); + const b = pairs.map(([, y]) => get(y)); + out[name] = { arm: round(median(a)), base: round(median(b)) }; + if (RATIOS.includes(name)) { + out[name].ratio = round( + median(pairs.map(([x, y]) => get(x) / get(y)).filter(Number.isFinite)), + 3 + ); + out[name].armCheaper = pairs.filter(([x, y]) => get(x) < get(y)).length; + } + } + return out; +} + +function compareArm(rows, arm, base) { + const index = new Map( + rows.filter((r) => r.arm === base).map((r) => [r.key, r]) + ); + const pairs = rows + .filter((r) => r.arm === arm && index.has(r.key)) + .map((r) => [r, index.get(r.key)]); + const build = pairs.filter(([a]) => a.kind === 'build'); + const plan = pairs.filter(([a]) => a.kind === 'plan'); + const section = (ps) => ({ + pairs: ps.length, + ...Object.fromEntries(BUILD_GROUPS.map((g) => [g, binary(ps, g)])), + }); + const categories = [...new Set(pairs.map(([a]) => a.category))].sort(); + return { + arm, + base, + build: section(build), + plan: plans(plan), + cost: { build: cost(build), plan: cost(plan) }, + byCategory: Object.fromEntries( + categories.map((c) => { + const b = build.filter(([a]) => a.category === c); + const p = plan.filter(([a]) => a.category === c); + return [ + c, + { + build: b.length + ? { + pairs: b.length, + all_green: binary(b, 'all_green'), + acceptance: binary(b, 'acceptance'), + constraints: binary(b, 'constraints'), + constraints_raw: binary(b, 'constraints_raw'), + } + : null, + plan: p.length ? plans(p) : null, + cost: (({ + first_turn_tokens, + context_tokens, + read_tokens, + processed_tokens, + wall_ms, + }) => ({ + first_turn_tokens, + context_tokens, + read_tokens, + processed_tokens, + wall_ms, + }))(cost([...b, ...p])), + }, + ]; + }) + ), + }; +} + +async function main() { + const o = options(process.argv.slice(2)); + let rows = await loadRows(o); + const present = [...new Set(rows.map((r) => r.arm))]; + const arms = ( + o.arms.length ? o.arms : ARMS.filter((a) => present.includes(a)) + ).filter((a) => a !== o.base); + const compared = new Set([o.base, ...arms]); + rows = rows.filter((r) => compared.has(r.arm)); + const flags = {}; + for (const r of rows) { + const f = (flags[r.arm] ??= { + sessions: 0, + excluded: 0, + left_folder: 0, + hidden_read: 0, + none_found_proof: 0, + timed_out: 0, + api_error: 0, + apply_failed: 0, + edited_existing_tests: 0, + edited_test_support: 0, + }); + f.sessions++; + for (const k of [ + 'excluded', + 'left_folder', + 'hidden_read', + 'none_found_proof', + 'timed_out', + ]) + if (r.grade.flags?.[k]) f[k]++; + if (r.session.api_error) f.api_error++; + if (r.grade.apply_failed) f.apply_failed++; + if (r.grade.edited_existing_tests?.length) f.edited_existing_tests++; + if (r.grade.edited_test_support?.length) f.edited_test_support++; + } + const dropped = new Set( + rows + .filter( + (r) => + (!o.keepFlagged && r.grade.flags?.excluded) || + (o.errors === 'drop' && r.session.api_error) + ) + .map((r) => r.key) + ); + rows = rows.filter((r) => !dropped.has(r.key)); + if (!rows.some((r) => r.arm === o.base)) + throw new Error( + `No graded ${o.base} sessions left in ${o.runs.join(', ')}${ + dropped.size + ? ` after dropping ${dropped.size} flagged (task, run) pairs: ${[ + ...dropped, + ].join(', ')} (see --keep-flagged)` + : '' + }` + ); + const comparisons = arms.map((arm) => compareArm(rows, arm, o.base)); + const pooled = Object.fromEntries( + comparisons + .filter((c) => PRIMARY.includes(c.arm) && c.build.pairs) + .map((c) => [c.arm, c.build.all_green.p]) + ); + const primary = {}; + for (const c of comparisons.filter((x) => PRIMARY.includes(x.arm))) + for (const category of PRIMARY_CATEGORIES) { + const build = c.byCategory[category]?.build; + if (build) primary[`${category}:${c.arm}`] = build.all_green.p; + } + const report = { + work: o.dir, + runs: o.runs, + split: o.split, + base: o.base, + flags, + droppedPairs: [...dropped].sort(), + holm_primary: holm(primary), + holm_all_green: holm(pooled), + comparisons, + ...(arms.includes('dump10x') && rows.some((r) => r.arm === 'dump') + ? { dump10x_vs_dump: compareArm(rows, 'dump10x', 'dump') } + : {}), + }; + const json = `${JSON.stringify(report, null, 2)}\n`; + if (o.out) await writeFile(resolve(o.out), json); + process.stdout.write(json); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); diff --git a/benchmarks/proof-build/scripts/gate.mjs b/benchmarks/proof-build/scripts/gate.mjs new file mode 100644 index 00000000..210ff1bb --- /dev/null +++ b/benchmarks/proof-build/scripts/gate.mjs @@ -0,0 +1,329 @@ +#!/usr/bin/env node +// The offline gate: free checks that every task must pass before any paid +// session runs (DESIGN.md §4). +// +// node scripts/gate.mjs --hidden