diff --git a/ROADMAP.md b/ROADMAP.md index 8bb2da01a..84efe9c6d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -772,7 +772,7 @@ graph LR | Web UI + macOS app UX audit | In progress | P0 | — | | | | Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | — | [081-release-qa-gate](./specs/081-release-qa-gate/) | | | Action log / transparency — info at a glance | In progress | P1 | — | | | -| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 33/64 (52%) | [103-token-bench](./specs/103-token-bench/) | #1141 | +| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 37/64 (58%) | [103-token-bench](./specs/103-token-bench/) | #1141 | | Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — | | | | Telemetry v7: honest funnel + churn instrumentation | In progress | P1 | — | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) | | | Planning/docs truth automation | In progress | P2 | — | | | @@ -914,4 +914,4 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—` | [100-prompt-rugpull-baseline](./specs/100-prompt-rugpull-baseline/) | — | — | | [101-tpa-db](./specs/101-tpa-db/) | — | — | | [102-schema-deferred](./specs/102-schema-deferred/) | `shipped` | 89/89 (100%) | -| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 33/64 (52%) | +| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 37/64 (58%) | diff --git a/bench/payloaddecomp.go b/bench/payloaddecomp.go new file mode 100644 index 000000000..e0aa8dd7b --- /dev/null +++ b/bench/payloaddecomp.go @@ -0,0 +1,312 @@ +package bench + +import ( + "encoding/json" + "fmt" +) + +// Spec 103 US4 — payload decomposition. +// +// Spec 102 projected ~70% savings from deferring input schemas and measured +// 29.7% on the frozen 45-tool corpus, concluding that names, descriptions and +// annotations carry most of the payload rather than schemas. That conclusion +// was drawn from two frozen corpora and never checked against another shape, +// and it is the single claim every reader of the benchmark will want tested. +// +// This file tests it the only way that can be honest: attribute the payload of +// a real corpus to its components and recompute the achievable ceiling FOR THAT +// CORPUS. +// +// Two measurement hazards shape the implementation, and both would produce a +// plausible-looking wrong answer: +// +// 1. TOKENS ARE NOT ADDITIVE ACROSS CONCATENATION. BPE merges across component +// boundaries, so Count(name) + Count(description) + Count(schema) does not +// equal Count(name + "\n" + description + "\n" + schema). Counting each +// component separately yields shares that never sum to the payload and that +// misattribute every boundary. So the payload is tokenized ONCE and each +// token is attributed to a component by byte offset, reusing the span +// partition AttributeTokens already implements for response cost. +// +// 2. THE CORPUS CARRIES NO ANNOTATIONS. bench.Tool is {ToolID, Server, Name, +// Description, Schema}. Spec 102 measured the production WIRE payload, +// which includes mcp-go's annotation defaults; the frozen corpora do not. +// These are different payloads, and a decomposition that silently reported +// a three-way split as though it settled a claim about four components +// would read as a confirmation while measuring something else. The +// Limitation field says so, in the report, next to the verdict. + +// PayloadComponent labels the parts of a rendered tool definition. +const ( + PayloadComponentName = "name" + PayloadComponentDescription = "description" + PayloadComponentSchema = "schema" + // PayloadComponentStructural is the rendering's own overhead — the + // separators between components. Small, but it belongs to some component + // or the shares do not partition the payload, and folding it into a + // content component would overstate that component. + PayloadComponentStructural = "structural" +) + +// PayloadDecomposition attributes one corpus's rendered payload to its +// components, and states what that corpus shape can and cannot settle. +type PayloadDecomposition struct { + FleetID string `json:"fleet_id"` + ToolCount int `json:"tool_count"` + TotalTokens int `json:"total_tokens"` + // ToolsWithSchema is how many of ToolCount actually carry an input schema. + // Load-bearing: a corpus with none cannot measure a schema share at all, + // and a 0% result there means "nothing to defer was present", not "schemas + // are cheap". Reporting the second from the first is the failure this + // field exists to make impossible. + ToolsWithSchema int `json:"tools_with_schema"` + + NameTokens int `json:"name_tokens"` + DescriptionTokens int `json:"description_tokens"` + SchemaTokens int `json:"schema_tokens"` + StructuralTokens int `json:"structural_tokens"` + + NamePct float64 `json:"name_pct"` + DescriptionPct float64 `json:"description_pct"` + SchemaPct float64 `json:"schema_pct"` + StructuralPct float64 `json:"structural_pct"` + + // SchemaCeilingPct is the most any schema-deferring design could ever save + // on THIS corpus: the schema share, since deleting the schema entirely is + // the upper bound of deferring it. Recomputed per corpus — carrying a + // ceiling forward is the error spec 102 made. + SchemaCeilingPct float64 `json:"schema_ceiling_pct"` + + // Spec102Verdict is an explicit confirmed/corrected statement rather than + // a number the reader has to interpret. + Spec102Verdict string `json:"spec_102_verdict"` + + // Limitation names what this corpus shape cannot settle. Always populated: + // a decomposition with no stated limits is claiming there are none. + Limitation string `json:"limitation"` + + Provenance string `json:"provenance"` +} + +// DecomposePayload attributes the canonical rendering of every tool in the +// corpus to name, description, schema and structural overhead. +// +// The rendering matches the baseline arm's canonical form — +// name + "\n" + description + "\n" + canonical(schema) — because that is what +// the savings figures are measured against. Decomposing a different rendering +// would produce shares that do not explain those figures. +func DecomposePayload(tk *Tokenizer, corpus *Corpus, fleetID string) (*PayloadDecomposition, error) { + if tk == nil { + return nil, fmt.Errorf("payload decomposition: nil tokenizer") + } + if corpus == nil || len(corpus.Tools) == 0 { + // Never a zero-filled result: zeros render as "measured, and every + // component costs nothing", which is worse than an error. + return nil, fmt.Errorf("payload decomposition: corpus %q has no tools", fleetID) + } + + d := &PayloadDecomposition{ + FleetID: fleetID, + ToolCount: len(corpus.Tools), + Provenance: ProvenanceMeasured, + } + + for _, tl := range corpus.Tools { + if len(tl.Schema) > 0 { + d.ToolsWithSchema++ + } + text, spans, err := partitionToolDefinition(tl) + if err != nil { + return nil, fmt.Errorf("payload decomposition: tool %s: %w", tl.ToolID, err) + } + total, components, err := AttributeTokens(tk, text, spans) + if err != nil { + return nil, fmt.Errorf("payload decomposition: tool %s: %w", tl.ToolID, err) + } + d.TotalTokens += total + d.NameTokens += components[PayloadComponentName] + d.DescriptionTokens += components[PayloadComponentDescription] + d.SchemaTokens += components[PayloadComponentSchema] + d.StructuralTokens += components[PayloadComponentStructural] + } + + if d.TotalTokens > 0 { + pct := func(n int) float64 { return float64(n) / float64(d.TotalTokens) * 100 } + d.NamePct = pct(d.NameTokens) + d.DescriptionPct = pct(d.DescriptionTokens) + d.SchemaPct = pct(d.SchemaTokens) + d.StructuralPct = pct(d.StructuralTokens) + d.SchemaCeilingPct = d.SchemaPct + } + + d.Spec102Verdict = spec102Verdict(d) + d.Limitation = "This decomposition covers the corpus rendering only: name, description and " + + "input schema. bench.Tool carries no annotations field, while spec 102 measured the " + + "production wire payload including mcp-go's annotation defaults. The annotations third " + + "of its conclusion is therefore OUT OF REACH of any corpus-based decomposition and is " + + "neither confirmed nor refuted here." + + return d, nil +} + +// spec102Verdict states what THIS CORPUS RENDERING shows, and is careful not to +// claim more than that. +// +// Two ways a naive verdict goes wrong here, both found by running the first +// version against the real corpora: +// +// 1. A CORPUS WITH NO SCHEMAS yields a 0% schema share, and a verdict phrased +// as "deferring schemas cannot exceed 0%" reads as a resounding +// confirmation of spec 102. It is nothing of the kind — it measures a +// corpus that does not contain the thing under test. The committed +// 527-tool snapshot is exactly this case: all 527 tools, zero schemas. +// +// 2. IT IS A DIFFERENT PAYLOAD FROM THE ONE SPEC 102 MEASURED. Spec 102 +// measured the production wire payload — mcp.Tool marshalling, the +// "[server] " prefix, mcp-go's annotation defaults, the raw-schema +// placeholder. This decomposition covers the corpus rendering, which has no +// annotations and no wire framing. A confident CONFIRMED/CORRECTED across +// that gap would be adjudicating a claim on evidence about something else. +// +// So the verdict describes the corpus and explicitly declines to settle spec +// 102's figure. The decomposition is still the useful thing — it says where the +// payload actually goes on a given fleet — it just is not a verdict on a +// measurement taken elsewhere. +func spec102Verdict(d *PayloadDecomposition) string { + prose := d.NamePct + d.DescriptionPct + const scope = " This describes the CORPUS RENDERING (name, description, schema). Spec 102's " + + "29.7% figure was measured on the production WIRE payload, which additionally carries " + + "annotations and wire framing, so this neither confirms nor corrects it." + + if d.ToolsWithSchema == 0 { + return fmt.Sprintf("INAPPLICABLE: none of the %d tools in this corpus carries an input "+ + "schema, so there is no schema share to measure and no ceiling to compute. A 0%% "+ + "result here means the corpus lacks the thing under test, NOT that schemas are "+ + "cheap.%s", d.ToolCount, scope) + } + + coverage := float64(d.ToolsWithSchema) / float64(d.ToolCount) * 100 + partial := "" + if d.ToolsWithSchema < d.ToolCount { + partial = fmt.Sprintf(" Note only %d of %d tools (%.0f%%) carry a schema, so the share is "+ + "diluted by the schema-less remainder.", d.ToolsWithSchema, d.ToolCount, coverage) + } + + switch { + case d.SchemaPct >= 50: + return fmt.Sprintf("SCHEMA-DOMINANT on this corpus: schemas are %.1f%% of the payload "+ + "against %.1f%% for names and descriptions, so deferring them could save up to "+ + "%.1f%% here.%s%s", d.SchemaPct, prose, d.SchemaCeilingPct, partial, scope) + case prose > d.SchemaPct: + return fmt.Sprintf("PROSE-DOMINANT on this corpus: names and descriptions are %.1f%% "+ + "against %.1f%% for schemas, so deferring schemas cannot exceed %.1f%% here however "+ + "it is implemented.%s%s", prose, d.SchemaPct, d.SchemaCeilingPct, partial, scope) + default: + return fmt.Sprintf("EVENLY SPLIT on this corpus: schemas %.1f%%, names plus descriptions "+ + "%.1f%%. Ceiling %.1f%%.%s%s", d.SchemaPct, prose, d.SchemaCeilingPct, partial, scope) + } +} + +// partitionToolDefinition builds the canonical rendering of one tool together +// with a span partition covering it exactly. +// +// The spans must be contiguous and cover [0, len(text)) — AttributeTokens +// enforces that — which is why the separators get their own structural spans +// rather than being folded into a neighbour. +func partitionToolDefinition(tl Tool) (string, []Span, error) { + var ( + text string + spans []Span + ) + + add := func(label, s string) { + if s == "" { + return + } + start := len(text) + text += s + spans = append(spans, Span{Label: label, Start: start, End: len(text)}) + } + + add(PayloadComponentName, tl.Name) + add(PayloadComponentStructural, "\n") + add(PayloadComponentDescription, tl.Description) + + if len(tl.Schema) > 0 { + canon, err := canonicalSchemaJSON(tl.Schema) + if err != nil { + return "", nil, err + } + add(PayloadComponentStructural, "\n") + add(PayloadComponentSchema, canon) + } + + if text == "" { + return "", nil, fmt.Errorf("tool renders to an empty definition") + } + return text, spans, nil +} + +// canonicalSchemaJSON matches the canonicalisation the baseline arm applies, so +// the decomposition explains the same payload the savings figures are measured +// against rather than a re-marshalled approximation of it. +// +// bench cannot import bench/arms (arms import bench), so the canonical form is +// reproduced here; the drift risk is bounded because both are "unmarshal into +// interface{}, re-marshal with sorted keys", which encoding/json does by +// construction for maps. +func canonicalSchemaJSON(raw json.RawMessage) (string, error) { + var v interface{} + if err := json.Unmarshal(raw, &v); err != nil { + return "", fmt.Errorf("schema is not valid JSON: %w", err) + } + out, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("schema could not be re-marshalled: %w", err) + } + return string(out), nil +} + +// PayloadDecompositionBlockFor builds the report block from one decomposition +// per fleet shape (T053). +// +// FR-024 asks for at least two shapes, because the whole point is to show how +// the attribution MOVES with fleet size rather than to publish a single-corpus +// projection — which is the shape of mistake spec 102 made. Fewer than two is +// therefore an error rather than a smaller block. +func PayloadDecompositionBlockFor(ds []*PayloadDecomposition) (*PayloadDecompositionBlock, error) { + if len(ds) < 2 { + return nil, fmt.Errorf("payload decomposition: %d fleet shape(s); FR-024 requires at "+ + "least two so the attribution can be seen to move with fleet size", len(ds)) + } + + block := &PayloadDecompositionBlock{ + AccountingSource: AccountingSource{ + Kind: AccountingKindTokenizer, + Identity: DefaultEncoding, + }, + } + + for _, d := range ds { + if d == nil { + return nil, fmt.Errorf("payload decomposition: nil shape") + } + block.Shapes = append(block.Shapes, PayloadDecompositionRow{ + FleetShape: FleetShape{ + ID: d.FleetID, + ToolCount: d.ToolCount, + }, + Provenance: d.Provenance, + ShareNamesPct: d.NamePct, + ShareDescriptionsPct: d.DescriptionPct, + // Deliberately nil: see the field's doc comment. The corpora carry + // no annotations, and a zero here would claim a measurement that + // was never taken. + ShareAnnotationsPct: nil, + ShareSchemasPct: d.SchemaPct, + AchievableCeilingPct: d.SchemaCeilingPct, + Spec102Verdict: d.Spec102Verdict, + }) + } + return block, nil +} diff --git a/bench/payloaddecomp_test.go b/bench/payloaddecomp_test.go new file mode 100644 index 000000000..37075a308 --- /dev/null +++ b/bench/payloaddecomp_test.go @@ -0,0 +1,271 @@ +package bench + +import ( + "encoding/json" + "strings" + "testing" +) + +// decompCorpus is a small schema-bearing corpus. The schemas are deliberately +// larger than the descriptions on one tool and smaller on another, so a test +// that only ever sees one shape dominating cannot pass by accident. +func decompCorpus() *Corpus { + return &Corpus{ + Version: "decomp_test@1", + Tools: []Tool{ + { + ToolID: "srv:big_schema", + Server: "srv", + Name: "big_schema", + Description: "Short.", + Schema: json.RawMessage(`{"type":"object","properties":{` + + `"alpha":{"type":"string","description":"the first parameter of several"},` + + `"beta":{"type":"integer","description":"the second parameter of several"},` + + `"gamma":{"type":"boolean","description":"the third parameter of several"}},` + + `"required":["alpha","beta"]}`), + }, + { + ToolID: "srv:big_description", + Server: "srv", + Name: "big_description", + Description: "A deliberately long description that runs well past the length of " + + "this tool's input schema, so the decomposition has at least one tool where " + + "prose rather than structure dominates the payload.", + Schema: json.RawMessage(`{"type":"object"}`), + }, + { + ToolID: "srv:no_schema", + Server: "srv", + Name: "no_schema", + Description: "Carries no schema at all.", + }, + }, + } +} + +// T050 — the shares must PARTITION the payload, not merely approximate it. +// +// This is the whole reason the decomposition attributes token spans over a +// single tokenization instead of counting each component separately. BPE merges +// across boundaries, so Count(name) + Count(description) + Count(schema) does +// NOT equal Count(name + description + schema). Independent counts would +// produce shares that look plausible, never sum, and quietly misattribute the +// bytes at every component boundary. +func TestPayloadDecomposition_SharesPartitionThePayload(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + + sum := d.NameTokens + d.DescriptionTokens + d.SchemaTokens + d.StructuralTokens + if sum != d.TotalTokens { + t.Errorf("shares must sum to the payload exactly: %d + %d + %d + %d = %d, want %d", + d.NameTokens, d.DescriptionTokens, d.SchemaTokens, d.StructuralTokens, sum, d.TotalTokens) + } + if d.TotalTokens == 0 { + t.Fatal("a non-empty corpus must have a non-zero payload") + } + for _, c := range []struct { + label string + n int + }{ + {"name", d.NameTokens}, + {"description", d.DescriptionTokens}, + {"schema", d.SchemaTokens}, + } { + if c.n <= 0 { + t.Errorf("%s share is %d; this corpus has all three, so a zero means the "+ + "attribution collapsed rather than measured", c.label, c.n) + } + } +} + +// T051 — the ceiling is a property of the corpus and must be recomputed for +// each one. +// +// Carrying a ceiling forward is precisely the error spec 102 made: it projected +// ~70% from an assumption about one corpus shape and measured 29.7%. A +// decomposition that reused a previous corpus's ceiling would repeat it while +// looking like a measurement. +func TestPayloadDecomposition_CeilingIsPerCorpus(t *testing.T) { + tk := runnerTokenizer(t) + + schemaHeavy := decompCorpus() + proseHeavy := &Corpus{ + Version: "prose@1", + Tools: []Tool{{ + ToolID: "srv:prose", Server: "srv", Name: "prose", + Description: strings.Repeat("a great deal of description text and nothing else. ", 40), + Schema: json.RawMessage(`{"type":"object"}`), + }}, + } + + a, err := DecomposePayload(tk, schemaHeavy, "a@1") + if err != nil { + t.Fatalf("DecomposePayload(schemaHeavy): %v", err) + } + b, err := DecomposePayload(tk, proseHeavy, "b@1") + if err != nil { + t.Fatalf("DecomposePayload(proseHeavy): %v", err) + } + + if a.SchemaCeilingPct == b.SchemaCeilingPct { + t.Errorf("two corpora with opposite shapes produced an identical ceiling (%.2f%%); "+ + "the ceiling must be recomputed per corpus, never carried forward", a.SchemaCeilingPct) + } + if b.SchemaCeilingPct >= a.SchemaCeilingPct { + t.Errorf("the prose-heavy corpus must have a LOWER schema ceiling than the "+ + "schema-heavy one: prose=%.2f%% schema=%.2f%%", b.SchemaCeilingPct, a.SchemaCeilingPct) + } + if b.SchemaCeilingPct < 0 || a.SchemaCeilingPct > 100 { + t.Errorf("ceiling out of range: a=%.2f b=%.2f", a.SchemaCeilingPct, b.SchemaCeilingPct) + } +} + +// The block must state what this corpus shape CANNOT settle. +// +// Spec 102 concluded that names, descriptions AND ANNOTATIONS dominate the +// payload. bench.Tool carries no annotations field, so a corpus-based +// decomposition cannot confirm or refute the annotations third of that claim. +// Reporting a verdict without saying so would look like a confirmation and +// would not be one. +func TestPayloadDecomposition_DeclaresTheAnnotationsLimit(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if !strings.Contains(strings.ToLower(d.Limitation), "annotation") { + t.Errorf("the decomposition must state that annotations are absent from the corpus "+ + "shape and therefore out of its reach; got %q", d.Limitation) + } + if d.Spec102Verdict == "" { + t.Error("the block must carry an explicit spec-102 verdict, not leave it implied") + } +} + +// An empty corpus is an error, not a zero-filled decomposition. +// +// Zeros would render as "measured, and everything costs nothing", which is the +// same silent-zero failure the exclusion accounting exists to prevent. +func TestPayloadDecomposition_EmptyCorpusIsAnError(t *testing.T) { + tk := runnerTokenizer(t) + + if _, err := DecomposePayload(tk, &Corpus{Version: "empty@1"}, "empty@1"); err == nil { + t.Error("an empty corpus must be an error, never a zero-filled decomposition") + } +} + +// A corpus with no schemas cannot measure a schema share, and must say so +// rather than reporting a confident 0%. +// +// Found by running the first version against the committed 527-tool snapshot: +// all 527 tools, zero schemas. The verdict read "deferring schemas cannot +// exceed 0.0% however it is implemented" — which sounds like a decisive result +// and is really a corpus that does not contain the thing under test. +func TestPayloadDecomposition_SchemalessCorpusIsInapplicable(t *testing.T) { + tk := runnerTokenizer(t) + + schemaless := &Corpus{ + Version: "schemaless@1", + Tools: []Tool{ + {ToolID: "s:a", Server: "s", Name: "a", Description: "no schema here"}, + {ToolID: "s:b", Server: "s", Name: "b", Description: "nor here"}, + }, + } + + d, err := DecomposePayload(tk, schemaless, "schemaless@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if d.ToolsWithSchema != 0 { + t.Fatalf("fixture should carry no schemas, got %d", d.ToolsWithSchema) + } + if !strings.Contains(d.Spec102Verdict, "INAPPLICABLE") { + t.Errorf("a schema-less corpus must yield an INAPPLICABLE verdict, not a confident "+ + "0%% result; got %q", d.Spec102Verdict) + } +} + +// The verdict must never claim to confirm or correct spec 102. +// +// Spec 102 measured the production wire payload — annotations and framing +// included. This decomposition covers the corpus rendering. Adjudicating that +// claim across the gap would be answering with evidence about something else, +// which is precisely what the Limitation field warns of; the verdict has to +// respect its own caveat. +func TestPayloadDecomposition_VerdictDeclinesToSettleSpec102(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + v := d.Spec102Verdict + for _, forbidden := range []string{"CONFIRMED", "CORRECTED"} { + if strings.Contains(v, forbidden) { + t.Errorf("the verdict must not claim to %s spec 102 from corpus evidence — it is a "+ + "different payload; got %q", forbidden, v) + } + } + if !strings.Contains(v, "WIRE payload") { + t.Errorf("the verdict must name the payload gap it is declining to cross; got %q", v) + } +} + +// T053 — the block must carry a populated accounting source, and must report +// the annotation share as NULL rather than zero. +// +// Zero would claim a measurement that was never taken. The corpora carry no +// annotations field at all, so "not measured" is the only truthful value, and +// the difference between null and 0 is the difference between an honest gap and +// a fabricated finding. +func TestPayloadDecompositionBlock_AnnotationShareIsNullNotZero(t *testing.T) { + tk := runnerTokenizer(t) + + a, err := DecomposePayload(tk, decompCorpus(), "a@1") + if err != nil { + t.Fatalf("DecomposePayload(a): %v", err) + } + b, err := DecomposePayload(tk, decompCorpus(), "b@1") + if err != nil { + t.Fatalf("DecomposePayload(b): %v", err) + } + + block, err := PayloadDecompositionBlockFor([]*PayloadDecomposition{a, b}) + if err != nil { + t.Fatalf("PayloadDecompositionBlockFor: %v", err) + } + if block.AccountingSource.IsZero() { + t.Error("the block must name a populated accounting source") + } + for i, row := range block.Shapes { + if row.ShareAnnotationsPct != nil { + t.Errorf("shape %d: the annotation share must be null (not measurable from a "+ + "corpus that carries no annotations), got %v", i, *row.ShareAnnotationsPct) + } + if row.Provenance != ProvenanceMeasured { + t.Errorf("shape %d: provenance %q", i, row.Provenance) + } + } +} + +// FR-024 wants at least two shapes, so a single-corpus block is an error. +// +// One shape is exactly the evidence base spec 102 projected from, and the point +// of the requirement is to make that shape of claim impossible to emit. +func TestPayloadDecompositionBlock_RequiresTwoShapes(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "only@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if _, err := PayloadDecompositionBlockFor([]*PayloadDecomposition{d}); err == nil { + t.Error("a single-shape decomposition block must be an error — one corpus is what " + + "spec 102 projected from") + } +} diff --git a/bench/reportv2.go b/bench/reportv2.go index 05adaf4f5..bffb328b4 100644 --- a/bench/reportv2.go +++ b/bench/reportv2.go @@ -500,8 +500,14 @@ type PayloadDecompositionRow struct { // The four attributions, as percentages of the definition payload. ShareNamesPct float64 `json:"share_names_pct"` ShareDescriptionsPct float64 `json:"share_descriptions_pct"` - ShareAnnotationsPct float64 `json:"share_annotations_pct"` - ShareSchemasPct float64 `json:"share_schemas_pct"` + // ShareAnnotationsPct is a POINTER because null and zero mean different + // things here. The frozen corpora carry no annotations field, so a + // corpus-based decomposition cannot measure this share at all — and a 0 + // would read as "measured, and annotations cost nothing", which is the + // silent-zero failure the rest of this report works to avoid. nil marshals + // to null: not measured. + ShareAnnotationsPct *float64 `json:"share_annotations_pct"` + ShareSchemasPct float64 `json:"share_schemas_pct"` // AchievableCeilingPct is recomputed for THIS shape. Carrying a // previously published ceiling forward is precisely the error spec 102 // made when it projected from a single corpus. diff --git a/bench/reportv2_test.go b/bench/reportv2_test.go index 8748b913a..894a717e5 100644 --- a/bench/reportv2_test.go +++ b/bench/reportv2_test.go @@ -382,6 +382,7 @@ func sampleAgentLoopBlock() *AgentLoopBlock { // samplePayloadDecomposition attributes definition cost across two fleet // shapes, each with its own recomputed ceiling and spec-102 verdict. func samplePayloadDecomposition() *PayloadDecompositionBlock { + annShare45, annShare527 := 1.5, 0.9 delta := -12.5 return &PayloadDecompositionBlock{ AccountingSource: AccountingSource{Kind: AccountingKindTokenizer, Identity: "cl100k_base"}, @@ -391,7 +392,7 @@ func samplePayloadDecomposition() *PayloadDecompositionBlock { Provenance: ProvenanceMeasured, ShareNamesPct: 3.1, ShareDescriptionsPct: 21.4, - ShareAnnotationsPct: 1.5, + ShareAnnotationsPct: &annShare45, ShareSchemasPct: 74.0, AchievableCeilingPct: 68.2, Spec102Verdict: Spec102Confirmed, @@ -401,7 +402,7 @@ func samplePayloadDecomposition() *PayloadDecompositionBlock { Provenance: ProvenanceMeasured, ShareNamesPct: 2.2, ShareDescriptionsPct: 30.8, - ShareAnnotationsPct: 0.9, + ShareAnnotationsPct: &annShare527, ShareSchemasPct: 66.1, AchievableCeilingPct: 55.7, Spec102Verdict: Spec102Corrected, diff --git a/specs/083-discovery-profiler/contracts/report-v2.schema.json b/specs/083-discovery-profiler/contracts/report-v2.schema.json index 4b79ea4d1..388d2b1d5 100644 --- a/specs/083-discovery-profiler/contracts/report-v2.schema.json +++ b/specs/083-discovery-profiler/contracts/report-v2.schema.json @@ -886,7 +886,11 @@ "type": "number" }, "share_annotations_pct": { - "type": "number" + "type": [ + "number", + "null" + ], + "$comment": "NULL means the annotation share was not measurable from this corpus shape, which is different from zero. The frozen corpora carry no annotations field, while spec 102 measured the production wire payload that does. Emitting 0 would read as 'measured, and annotations cost nothing' — the silent-zero failure the exclusion accounting exists to prevent." }, "share_schemas_pct": { "type": "number" diff --git a/specs/103-token-bench/tasks.md b/specs/103-token-bench/tasks.md index 75daea902..94d825a37 100644 --- a/specs/103-token-bench/tasks.md +++ b/specs/103-token-bench/tasks.md @@ -264,16 +264,16 @@ recomputed per corpus and an explicit `confirmed`/`corrected` verdict. ### Tests for User Story 4 ⚠️ -- [ ] T050 [US4] In `bench/payloaddecomp_test.go`: the four shares sum to the whole payload -- [ ] T051 [US4] In `bench/payloaddecomp_test.go`: the achievable ceiling is recomputed per +- [x] T050 [US4] In `bench/payloaddecomp_test.go`: the four shares sum to the whole payload +- [x] T051 [US4] In `bench/payloaddecomp_test.go`: the achievable ceiling is recomputed per corpus and never carried forward as a constant — that carry-forward is precisely the error spec 102 made. Same file as the task above, so not parallel ### Implementation for User Story 4 -- [ ] T052 [US4] Implement the decomposition in `bench/payloaddecomp.go` over at least two fleet +- [x] T052 [US4] Implement the decomposition in `bench/payloaddecomp.go` over at least two fleet shapes (the 45-tool reference corpus and the 527-tool snapshot) -- [ ] T053 [US4] Emit the `payload_decomposition` block from `bench/payloaddecomp.go` with a +- [x] T053 [US4] Emit the `payload_decomposition` block from `bench/payloaddecomp.go` with a populated `accounting_source` and an explicit `spec102_verdict` (`confirmed` or `corrected`, with the delta)