diff --git a/ROADMAP.md b/ROADMAP.md index 8bb2da01a..84efe9c6d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -772,7 +772,7 @@ graph LR | Web UI + macOS app UX audit | In progress | P0 | — | | | | Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | — | [081-release-qa-gate](./specs/081-release-qa-gate/) | | | Action log / transparency — info at a glance | In progress | P1 | — | | | -| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 33/64 (52%) | [103-token-bench](./specs/103-token-bench/) | #1141 | +| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 37/64 (58%) | [103-token-bench](./specs/103-token-bench/) | #1141 | | Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — | | | | Telemetry v7: honest funnel + churn instrumentation | In progress | P1 | — | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) | | | Planning/docs truth automation | In progress | P2 | — | | | @@ -914,4 +914,4 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—` | [100-prompt-rugpull-baseline](./specs/100-prompt-rugpull-baseline/) | — | — | | [101-tpa-db](./specs/101-tpa-db/) | — | — | | [102-schema-deferred](./specs/102-schema-deferred/) | `shipped` | 89/89 (100%) | -| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 33/64 (52%) | +| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 37/64 (58%) | diff --git a/bench/cmd/bench/main.go b/bench/cmd/bench/main.go index 5d951bfea..118bdb98e 100644 --- a/bench/cmd/bench/main.go +++ b/bench/cmd/bench/main.go @@ -35,10 +35,19 @@ // go run ./bench/cmd/bench -replay /path/outside/the/repo/activity.jsonl \ // -corpus-v2 specs/083-discovery-profiler/datasets/corpus_v2.tools.json -out bench/results // -// The fleet input is not optional. A menu is a property of the tool -// definitions and the activity export carries no fleet snapshot, so a -// recording on its own computes nothing — `-replay` without `-corpus-v2` or -// `-livemcptool` is an error, not a degraded run. +// The fleet may instead be read live from a running proxy's direct MCP +// surface, so an operator can price their OWN fleet without exporting a corpus +// first (Spec 103): +// +// go run ./bench/cmd/bench -replay /path/outside/the/repo/activity.jsonl \ +// -proxy http://127.0.0.1:8080 -api-key KEY -out bench/results +// +// The fleet input is not optional, and it is not plural. A menu is a property +// of the tool definitions and the activity export carries no fleet snapshot, +// so a recording on its own computes nothing — `-replay` with no fleet flag at +// all is an error, not a degraded run. Supplying TWO (say -corpus-v2 and +// -proxy) is also an error: every figure is quoted for exactly one fleet +// shape, and silently preferring one would make the reported fleet_id false. // // A LAP lint artifact (`uvx --from lap-score==0.8.0 lap lint --json`) merges // into either report via -lap-json. Reports land in bench/results/ @@ -91,10 +100,22 @@ func main() { // the EXISTING corpus flags (-corpus-v2 / -livemcptool) rather than a new // one: a replay fleet is a tool corpus, and a second way to name one would // be a second loader to keep honest. - replayPath := flag.String("replay", "", "path to an activity JSONL export (mcpproxy activity export --format json) to recompute cost over; REQUIRES a fleet input (-corpus-v2 or -livemcptool) — a recording alone computes nothing. The file must live OUTSIDE this repository and is never committed") + replayPath := flag.String("replay", "", "path to an activity JSONL export (mcpproxy activity export --format json) to recompute cost over; REQUIRES exactly one fleet input (-corpus-v2, -livemcptool, or -proxy to read a running proxy's own catalog over MCP tools/list) — a recording alone computes nothing, and two fleets is an error. The file must live OUTSIDE this repository and is never committed") replayBodies := flag.Bool("replay-bodies-on-unmasked", false, "replay: read recorded request/response bodies. OFF by default. The activity export path does NOT mask, so this reads raw unmasked user traffic; bodies are tokenized inside the loader and only counts leave it") flag.Parse() + // -proxy carries a non-empty DEFAULT, so its value cannot tell a replay + // run whether an operator actually asked for a live fleet. flag.Visit + // reports only what was set on the command line, which is the difference + // between "read this proxy's catalog" and "a default nobody typed" — + // without it, every -replay run would try to open a socket. + proxySet := false + flag.Visit(func(f *flag.Flag) { + if f.Name == "proxy" { + proxySet = true + } + }) + // -live and -replay are different measurements over different inputs, and // -live is checked first. Accepting both would silently run live mode and // discard -replay with no output and no warning — a benchmark that quietly @@ -132,6 +153,9 @@ func main() { outDir: *outDir, corpusV2Path: *corpusV2Path, livemcptoolPath: *livemcptoolPath, + liveProxy: *proxy, + liveProxySet: proxySet, + apiKey: *apiKey, }) return } @@ -474,6 +498,14 @@ type replayCLIOptions struct { outDir string corpusV2Path string livemcptoolPath string + + // liveProxy is the base URL a live fleet is read from, and liveProxySet + // records whether the operator actually typed -proxy. The two are separate + // because -proxy has a default: the URL alone cannot distinguish "read + // this proxy" from "nobody asked for a live fleet". + liveProxy string + liveProxySet bool + apiKey string } // runReplay is the Spec 103 US1 entry point: recompute per-cell menu cost and @@ -491,6 +523,15 @@ func runReplay(opts replayCLIOptions) { fleet, fleetID := loadReplayFleet(opts.corpusV2Path, opts.livemcptoolPath) + // The live fleet source is CONSTRUCTED here but not read: RunReplay + // resolves it, so "exactly one fleet input" and "a live fetch failure is + // fatal" are decided in one place (bench.ResolveFleet) rather than being + // half-implemented by every caller. + var liveFleet bench.FleetSource + if opts.liveProxySet { + liveFleet = bench.NewLiveFleetSource(context.Background(), opts.liveProxy, opts.apiKey) + } + // Resolve exactly the arms the matrix needs to price a menu. A missing arm // is fatal rather than a skipped row: an absent cell reads as "not // applicable" when the truth would be "we could not measure it". @@ -512,6 +553,7 @@ func runReplay(opts replayCLIOptions) { RecordingPath: opts.recording, Fleet: fleet, FleetID: fleetID, + LiveFleet: liveFleet, Bodies: bodies, Encoding: opts.encoding, Arms: armsByName, @@ -535,12 +577,18 @@ func runReplay(opts replayCLIOptions) { fmt.Fprintf(os.Stderr, "bench: the replay input is raw recorded traffic — delete %s when this run is finished\n", opts.recording) } -// loadReplayFleet resolves the fleet input from the existing corpus flags. +// loadReplayFleet resolves the FROZEN fleet input from the corpus flags. // // Neither flag returns a nil fleet on purpose: the refusal belongs to // bench.RunReplay, whose error explains WHY a recording alone computes -// nothing. Supplying both is refused here, because silently preferring one +// nothing. Two frozen corpora is refused here, because silently preferring one // would make the reported fleet shape a coin toss. +// +// The LIVE fleet input (-proxy) is deliberately not resolved here. Frozen +// against live is the same ambiguity as frozen against frozen, but it is +// decided by bench.ResolveFleet so that the rule — and the decision to treat a +// failed live fetch as fatal rather than as grounds to fall back — is stated +// once, in the package a test can reach without a CLI. func loadReplayFleet(corpusV2Path, livemcptoolPath string) (*bench.Corpus, string) { if corpusV2Path != "" && livemcptoolPath != "" { log.Fatalf("bench: replay takes ONE fleet input; -corpus-v2 and -livemcptool were both given, and every figure is quoted for exactly one fleet shape") diff --git a/bench/live_report.go b/bench/live_report.go index 31b61f330..75d791ce2 100644 --- a/bench/live_report.go +++ b/bench/live_report.go @@ -512,7 +512,14 @@ func isStubSchema(raw json.RawMessage) bool { Properties map[string]json.RawMessage `json:"properties"` } if err := json.Unmarshal(raw, &s); err != nil { - return false + // A schema that will not decode as an object carries no properties to + // price, so it is a stub for costing purposes. Returning false here — + // as an earlier version did — let a schema of `true`, `[]` or `0` pass + // as REAL and be priced at about one token, which is the + // semantically-empty-but-syntactically-different bypass around every + // guard downstream. Not reachable from mcpproxy today, which emits + // object schemas, but the guard should not depend on that. + return true } return len(s.Properties) == 0 } diff --git a/bench/mcpcaller.go b/bench/mcpcaller.go index d333445af..6ce6be43c 100644 --- a/bench/mcpcaller.go +++ b/bench/mcpcaller.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sync/atomic" "github.com/mark3labs/mcp-go/client" "github.com/mark3labs/mcp-go/client/transport" @@ -16,8 +17,25 @@ import ( // through the SAME pipeline against the same live index. The serialized // response text it returns is exactly what an agent would receive — that is // what gets tokenized. +// +// Spec 103 gave it a second job — ToolsListPage, the raw tools/list fetch the +// live fleet source is built on — because the handshake is the same one and a +// second connect+initialize would be a second thing to keep correct. The two +// jobs share the session; only ToolsListPage needs the transport handle. type MCPRetrieveCaller struct { client *client.Client + + // trans is the SAME transport the client wraps, kept so tools/list can be + // issued at the JSON-RPC level. That is not a shortcut: mcp-go's typed + // decode reshapes input schemas (it drops unknown top-level keywords and + // always re-emits "properties"/"required"), which would erase the exact + // bytes bench/mcptools.go's stub guard reads. See mcptools.go's header. + trans transport.Interface + + // rawID numbers the requests issued outside the client. The ids are + // STRINGS in their own namespace so they can never collide with the + // client's int64 counter on the shared session. + rawID atomic.Int64 } // NewMCPRetrieveCaller connects and initializes an MCP client against mcpURL @@ -29,6 +47,12 @@ func NewMCPRetrieveCaller(ctx context.Context, mcpURL string) (*MCPRetrieveCalle } c := client.NewClient(httpTransport) if err := c.Start(ctx); err != nil { + // Close on this path too, symmetric with the Initialize failure below. + // Benign with the current StreamableHTTP transport, whose Start is a + // no-op without continuous listening — but the symmetry is what stops + // a listener goroutine leaking the day that option is enabled, and + // noticing it then would be much harder than writing it now. + _ = c.Close() return nil, fmt.Errorf("mcp start %q: %w", mcpURL, err) } @@ -39,7 +63,7 @@ func NewMCPRetrieveCaller(ctx context.Context, mcpURL string) (*MCPRetrieveCalle _ = c.Close() return nil, fmt.Errorf("mcp initialize %q: %w", mcpURL, err) } - return &MCPRetrieveCaller{client: c}, nil + return &MCPRetrieveCaller{client: c, trans: httpTransport}, nil } // Close shuts the underlying MCP client down. @@ -47,6 +71,44 @@ func (m *MCPRetrieveCaller) Close() error { return m.client.Close() } +// ToolsListPage implements ToolsListPageFunc (mcptools.go): one tools/list +// round trip on the live session, returning the RAW JSON-RPC result. +// +// It goes through the transport rather than client.ListTools deliberately. +// ListTools returns a decoded *mcp.ListToolsResult, and mcp-go's +// ToolArgumentsSchema keeps only {$defs, type, properties, required, +// additionalProperties} while its marshaller always re-emits "properties" and +// "required" — so re-serializing a decoded result rewrites the Spec 102 +// deferred placeholder {"type":"object"} into +// {"properties":{},"required":[],"type":"object"} and makes it indistinguish- +// able from a genuinely parameter-less tool. The stub guard exists precisely +// to tell those apart, so the raw bytes are load-bearing, not an optimization. +// +// Pagination is NOT handled here: this is one page, and the cursor loop lives +// in FetchToolsList where it is testable without a proxy. +func (m *MCPRetrieveCaller) ToolsListPage(ctx context.Context, cursor string) (json.RawMessage, error) { + params := struct { + Cursor string `json:"cursor,omitempty"` + }{Cursor: cursor} + + resp, err := m.trans.SendRequest(ctx, transport.JSONRPCRequest{ + JSONRPC: mcp.JSONRPC_VERSION, + ID: mcp.NewRequestId(fmt.Sprintf("bench-tools-list-%d", m.rawID.Add(1))), + Method: MCPToolsListMethod, + Params: params, + }) + if err != nil { + return nil, fmt.Errorf("%s: %w", MCPToolsListMethod, err) + } + if resp.Error != nil { + return nil, fmt.Errorf("%s: server error %d: %s", MCPToolsListMethod, resp.Error.Code, resp.Error.Message) + } + if len(resp.Result) == 0 { + return nil, fmt.Errorf("%s: server returned an empty result", MCPToolsListMethod) + } + return resp.Result, nil +} + // RetrieveFunc adapts the caller to the RetrieveToolsFunc signature RunFlipGates // consumes. func (m *MCPRetrieveCaller) RetrieveFunc(ctx context.Context) RetrieveToolsFunc { diff --git a/bench/mcptools.go b/bench/mcptools.go new file mode 100644 index 000000000..3cae00f12 --- /dev/null +++ b/bench/mcptools.go @@ -0,0 +1,534 @@ +// mcptools.go — the live MCP tools/list primitive (Spec 103). +// +// # What this is for, and what it is NOT for +// +// The obvious justification for reading tools/list is "REST serves stub +// schemas". On this branch that is FALSE and must not be repeated: GET +// /api/v1/tools returns real upstream input schemas (MCP-3167 is fixed in +// internal/contracts/converters.go, which now reads "inputSchema" first and +// falls back to the legacy "schema"). Measured against the 7-server reference +// fleet, 42 of 45 REST schemas are real and the 3 empties are genuinely +// parameter-less tools serving identical bytes on both surfaces. +// +// The real justification is FIDELITY. tools/list is the only surface that +// shows what an agent actually receives: +// +// - it is the MENU, not a catalog. REST /api/v1/tools is a different +// population (it skips quarantined servers but includes disabled servers +// and non-callable tools) and a different rendering (no "[server] " +// prefix, no deferred signature suffix). +// - it reflects the deployment. bench's in-process derivation +// (internal/server.ProxyModeToolDefs) pins EnableCodeExecution:true +// unconditionally, so it always counts code_execution's 2420-char +// description even for a deployment with code execution off — the shipped +// default — overstating the proxy menu by 1049 tokens. +// - it is verifiably the same arithmetic. A captured tools/list run through +// bench's own tokenizer reproduces the in-process built-in menu EXACTLY +// (5783 tokens for the retrieve_tools surface, 3781 for code_execution). +// +// # The stub guard is the point of this file +// +// A fleet of stub schemas prices every upstream definition at a fraction of +// its real size. That shrinks the naive baseline, which is the denominator of +// every published savings percentage, so the error lands as INFLATED SAVINGS — +// the one direction of error this benchmark exists to prevent. Two systematic +// sources are known: +// +// 1. The Spec 102 deferred direct surface. With +// `direct_tool_response_mode: "deferred"`, /mcp/all serves the placeholder +// `{"type":"object"}` for EVERY upstream tool (internal/server +// deferredDirectInputSchema / renderDeferredDirectTool) and folds the real +// schema into the description as a compact signature. It is served with no +// error and no flag, and REST keeps serving real schemas under the very +// same config — so this is the live hazard, and it is silent. +// +// Measured end to end on the 7-server / 45-tool reference fleet, with this +// guard removed: direct_full's menu collapses from 4368 to 2502 tokens +// (-42.7%), and the published headline — the direct_full <-> direct_deferred +// delta — FLIPS SIGN, from +1300 tokens (29.8% saved) to -806 (-32.2%, +// i.e. deferral reported as COSTING more). That is the failure this file +// exists to make impossible: not a slightly-off number, but a report that +// asserts the opposite of the truth while looking entirely well-formed. +// +// 2. The supervisor StateView stub `{"type":"object","properties":{}}` +// (MCP-3132/MCP-3167). Fixed today; the guard is what keeps a regression +// from silently re-inflating the headline. +// +// The guard is stated per POPULATION rather than per tool, because a +// per-tool rule would be useless: real fleets contain genuinely parameter-less +// tools whose honest schema is `{"properties":{},"required":[],"type":"object"}`, +// and the deferred surface leaves describe_tool's schema real, so "does +// anything here have a schema" waves the deferred case straight through. The +// rule is instead: within each population PRESENT in the listing — upstream +// tools and mcpproxy's own built-ins — at least one tool must carry a schema +// with real properties. Both known failure modes stub an entire population; +// no healthy fleet stubs one. +// +// # Why the RAW wire bytes +// +// The schema must come off the wire as json.RawMessage, never through +// mcp-go's typed decode. ToolArgumentsSchema keeps only +// {$defs, type, properties, required, additionalProperties} and drops every +// other top-level keyword, and its marshaller ALWAYS emits "properties" and +// "required" — so a round trip rewrites the deferred placeholder +// `{"type":"object"}` as `{"properties":{},"required":[],"type":"object"}` and +// erases the exact distinction the guard depends on. The transport-level path +// is therefore not an optimization; it is what makes the guard possible. +// +// # The "[server] " prefix is stripped +// +// The direct surface renders every description as `fmt.Sprintf("[%s] %s", ...)` +// (internal/server/mcp_routing.go). Isolated on the reference fleet, that +// prefix costs 136 tokens across 45 tools — 3.0% (4595 with, 4459 without). +// It is stripped here, and the reason is not that 3% is small: the encoding +// arms SYNTHESIZE each cell's rendering from a canonical definition, and +// bench/arms.DirectDeferredArm prepends that same "[server] " itself. Keeping +// the prefix would double-charge it in the direct cells and make a live fleet +// non-interchangeable with every frozen corpus the published figures use. +// +// # Known limitation: server attribution is a first-"__" split +// +// Attribution uses the production parser (server.ParseDirectToolName), which +// splits on the FIRST "__" and is knowingly ambiguous: a server whose name +// contains "__" mis-splits. The proxy resolves this with server-side state +// (mcp_direct_catalog.go keeps byDisplayName AND byCanonical maps) that is not +// recoverable over the wire, and it WITHHOLDS both tools on a display-name +// collision rather than picking a winner — so a live listing can also under- +// count relative to GET /api/v1/tools. Reconciling a fleet's count against the +// REST catalog is the fix for that, and is deliberately left to the caller +// rather than smuggled in here. +package bench + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/server" +) + +// MCPToolsListMethod is the JSON-RPC method this primitive calls. +const MCPToolsListMethod = "tools/list" + +// deferredDirectPlaceholderSchema mirrors internal/server's +// deferredDirectInputSchema byte for byte. Matching the exact bytes is what +// lets the guard say "this is Spec 102 deferral" rather than the vaguer "these +// schemas look empty" — and an actionable error names the config to change. +const deferredDirectPlaceholderSchema = `{"type":"object"}` + +// maxToolsListPages bounds the pagination loop. A server that returns a +// cursor forever would otherwise hang a benchmark run with no output; the cap +// is far above any real fleet (a page is typically the whole catalog). +const maxToolsListPages = 512 + +// ErrStubToolSchemas is the guard's sentinel: a tools/list listing whose +// schemas cannot be priced. It is a sentinel rather than a formatted string so +// a caller (and a test) can assert the RULE — a stub-schema listing is refused, +// never returned — independently of the wording. +var ErrStubToolSchemas = errors.New("mcp tools/list served stub input schemas") + +// ErrReplayFleetAmbiguous is the two-fleets refusal. Silently preferring one +// input over the other would make the report's fleet_id a lie: every figure is +// quoted for exactly one fleet shape, and the reader has no way to tell which. +var ErrReplayFleetAmbiguous = errors.New("replay takes exactly one fleet input") + +// ToolsListPageFunc performs ONE tools/list round trip and returns the RAW +// JSON-RPC result object for that page (the `{"tools":[...],"nextCursor":...}` +// value, not the envelope). +// +// It is a function type for the reason RetrieveToolsFunc is (flipgate.go): the +// arithmetic that matters — identity mapping, prefix stripping, canonical- +// ization and above all the stub guard — must be testable with no proxy +// running, and a concrete client in the signature would pin every test to one. +// The raw json.RawMessage return is load-bearing; see the file header. +type ToolsListPageFunc func(ctx context.Context, cursor string) (json.RawMessage, error) + +// toolsListPage is the decoded shape of one tools/list result. The schema +// stays raw all the way through. +type toolsListPage struct { + Tools []wireTool `json:"tools"` + NextCursor string `json:"nextCursor"` +} + +// wireTool is one tools/list entry, decoded only as far as the fields that +// occupy an agent's context. Annotations, _meta, outputSchema and icons are +// deliberately not decoded: this primitive feeds bench.Tool, which models a +// definition as name + description + input schema, and inventing fields the +// arms cannot render would price a fiction. +type wireTool struct { + Name string `json:"name"` + Description string `json:"description"` + InputSchema json.RawMessage `json:"inputSchema"` +} + +// FetchToolsList performs a full, paginated MCP tools/list against one surface +// and maps the result onto []Tool with REAL schemas, or fails. +// +// surface is used only for diagnostics (e.g. "/mcp/all"); the transport is +// supplied by page. The returned order is the wire order, page by page, so two +// runs against an unchanged proxy produce identical fleets. +func FetchToolsList(ctx context.Context, surface string, page ToolsListPageFunc) ([]Tool, error) { + if page == nil { + return nil, fmt.Errorf("mcp tools/list %s: no transport supplied", surface) + } + + var wire []wireTool + cursor := "" + seenCursor := map[string]bool{} + for i := 0; ; i++ { + if i >= maxToolsListPages { + return nil, fmt.Errorf("mcp tools/list %s: stopped after %d pages — the server keeps returning a nextCursor", surface, maxToolsListPages) + } + raw, err := page(ctx, cursor) + if err != nil { + return nil, fmt.Errorf("mcp tools/list %s: %w", surface, err) + } + var decoded toolsListPage + if err := json.Unmarshal(raw, &decoded); err != nil { + return nil, fmt.Errorf("mcp tools/list %s: decode page %d: %w", surface, i, err) + } + wire = append(wire, decoded.Tools...) + if decoded.NextCursor == "" { + break + } + // A repeated cursor is a server bug that would otherwise duplicate the + // whole catalog into the fleet and double its menu cost. + if seenCursor[decoded.NextCursor] { + return nil, fmt.Errorf("mcp tools/list %s: server repeated cursor %q — pagination would loop", surface, decoded.NextCursor) + } + seenCursor[decoded.NextCursor] = true + cursor = decoded.NextCursor + } + + if len(wire) == 0 { + return nil, fmt.Errorf("mcp tools/list %s: the listing contains no tools — a zero-tool menu would be reported as a measurement of zero cost", surface) + } + + tools := make([]Tool, 0, len(wire)) + seenID := make(map[string]bool, len(wire)) + for i := range wire { + tl, err := toolFromWire(wire[i]) + if err != nil { + return nil, fmt.Errorf("mcp tools/list %s: entry %d (%s): %w", surface, i, wire[i].Name, err) + } + // The direct catalog withholds BOTH tools on a display-name collision + // rather than picking a winner, so a duplicate arriving here means + // something upstream is wrong. Counting it twice would inflate the menu. + if seenID[tl.ToolID] { + return nil, fmt.Errorf("mcp tools/list %s: duplicate tool id %q — counting it twice would inflate the menu cost", surface, tl.ToolID) + } + seenID[tl.ToolID] = true + tools = append(tools, tl) + } + + if err := guardToolSchemas(surface, tools); err != nil { + return nil, err + } + return tools, nil +} + +// toolFromWire maps one tools/list entry onto a canonical bench.Tool. +// +// Identity follows the conventions the rest of bench already uses, so a live +// fleet joins by id against a frozen corpus and an activity export without a +// translation layer: an upstream tool is ":" (matching +// LiveClient.FetchUpstreamTools and corpus_v2), and a tool with no "__" +// separator is one of mcpproxy's own, keyed "mcpproxy:" exactly as +// ProxyToolsForMode keys it. A prefix-less name is NEVER given a server of "" +// dressed up as attribution. +func toolFromWire(w wireTool) (Tool, error) { + schema, err := normalizeSchema(w.InputSchema) + if err != nil { + return Tool{}, err + } + + serverName, toolName, ok := server.ParseDirectToolName(w.Name) + if !ok { + return Tool{ + ToolID: "mcpproxy:" + w.Name, + Name: w.Name, + Description: w.Description, + Schema: schema, + }, nil + } + return Tool{ + ToolID: serverName + ":" + toolName, + Server: serverName, + Name: toolName, + Description: stripDirectServerPrefix(w.Description, serverName), + Schema: schema, + }, nil +} + +// stripDirectServerPrefix removes the "[server] " the direct surface prepends, +// and ONLY when it names this tool's own server — a description that merely +// opens with some other bracketed word is real text the agent pays for. See +// the file header for why stripping is the right call (the arms re-synthesize +// the prefix themselves) and for the 136-token measurement of what it costs. +// KNOWN CONSEQUENCE of the "__" mis-split documented in the file header: when a +// server name itself contains "__" (wire name "my__server__read"), the parsed +// server is "my", so this looks for "[my] " and leaves "[my__server] " in the +// description. The arms then re-synthesize their own prefix on top, and that +// tool is charged for the prefix TWICE. +// +// It is left rather than patched because a heuristic that guessed where the +// server name ends would mis-split a different set of names, and the cost is a +// few tokens on one tool rather than a wrong headline. The file header +// documents the mis-split's effect on attribution and counting; this is its +// effect on cost, recorded so nobody rediscovers it as a mystery discrepancy. +func stripDirectServerPrefix(description, serverName string) string { + prefix := "[" + serverName + "] " + return strings.TrimPrefix(description, prefix) +} + +// schemaCensus counts one population's schemas by kind. +type schemaCensus struct { + tools int + real int + placeholder int // exactly the Spec 102 deferred placeholder bytes + parameterless int // an explicit, honest "takes no arguments" + example string +} + +func (c *schemaCensus) add(t Tool) { + c.tools++ + switch { + case len(t.Schema) > 0 && !isStubSchema(t.Schema): + c.real++ + default: + switch { + case string(t.Schema) == deferredDirectPlaceholderSchema: + c.placeholder++ + case isParameterlessSchema(t.Schema): + // An honest parameter-less tool. list_allowed_directories, + // read_graph and list_tables all legitimately take no arguments, + // and they serve the SAME bytes on REST and MCP. Counting them as + // evidence of stubbing would reject a small real fleet with a + // misleading cause. + c.parameterless++ + } + if c.example == "" { + c.example = fmt.Sprintf("%s -> %q", t.ToolID, string(t.Schema)) + } + } +} + +// isParameterlessSchema reports whether raw is an explicit "this tool takes no +// arguments" declaration rather than a placeholder standing in for one. +// +// The distinction is the difference between refusing a real fleet and admitting +// a stubbed one. A parameter-less tool declares its emptiness — an explicit +// properties object, usually alongside an empty required list. The Spec 102 +// deferred placeholder and the old supervisor stub declare nothing at all. +func isParameterlessSchema(raw json.RawMessage) bool { + var probe struct { + Type string `json:"type"` + Properties *map[string]json.RawMessage `json:"properties"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return false + } + return probe.Type == "object" && probe.Properties != nil && len(*probe.Properties) == 0 +} + +// guardToolSchemas refuses a listing whose schemas cannot be priced. +// +// See the file header for the full rationale. In short: a stub-schema fleet +// shrinks the savings denominator and therefore INFLATES the headline, so it +// must fail loudly rather than flow into a report. The check is per population +// because a per-tool rule would refuse every real fleet (parameter-less tools +// exist) and a whole-listing rule would pass the deferred case (describe_tool +// keeps a real schema next to 45 placeholders). +func guardToolSchemas(surface string, tools []Tool) error { + var upstream, builtin schemaCensus + for _, t := range tools { + if t.Server == "" { + builtin.add(t) + continue + } + upstream.add(t) + } + + for _, pop := range []struct { + label string + census schemaCensus + }{ + {"upstream", upstream}, + {"built-in", builtin}, + } { + c := pop.census + // An ABSENT population is not an empty one: /mcp lists no upstream + // tools at all, and that is a fact about the surface, not a defect. + if c.tools == 0 || c.real > 0 { + continue + } + // AN EMPTY-PROPERTIES SCHEMA IS AMBIGUOUS BY SHAPE, and the error must + // say so rather than blaming a bug that may not be present. + // `{"type":"object","properties":{}}` is what a stubbed surface serves + // AND what an honestly parameter-less tool serves — + // filesystem:list_allowed_directories and memory:read_graph emit + // exactly those bytes on both REST and MCP. No inspection separates + // them. + // + // The population is still REFUSED, because the two failure directions + // are not symmetric: admitting a stubbed fleet prices its schemas at + // nothing and INFLATES the headline, which is the error class this + // whole benchmark exists to prevent, while refusing an honest + // parameter-less fleet costs its operator one clear message and a + // documented way forward. So refuse, and name the ambiguity. + // + // The Spec 102 deferred placeholder is NOT ambiguous — it carries no + // properties key at all — and gets its own, definite cause below. + cause := "every schema declares no properties. That shape is AMBIGUOUS: it is what a stubbed " + + "surface serves and equally what an honestly parameter-less tool serves, and nothing in the " + + "schema separates them. This is refused rather than admitted because the two directions are " + + "not symmetric — pricing stubbed schemas at nothing inflates the headline, while refusing a " + + "genuinely parameter-less fleet costs you this message. If your fleet really is all " + + "parameter-less, supply it as a frozen corpus (-corpus-v2) where the shape is a deliberate, " + + "reviewable input rather than something read off a live surface" + if c.placeholder > 0 { + cause = fmt.Sprintf("%d of them are exactly the Spec 102 placeholder %s, which means the proxy is running with "+ + "direct_tool_response_mode: \"deferred\" — that surface folds the real schema into the description, so a fleet read "+ + "from it under-counts the direct menu by ~40%% and FLIPS THE SIGN of the direct_full <-> direct_deferred delta "+ + "(measured on a 45-tool fleet: +1300 tokens / 29.8%% becomes -806 / -32.2%%). Set direct_tool_response_mode to \"full\" "+ + "and re-run", c.placeholder, deferredDirectPlaceholderSchema) + } + return fmt.Errorf("%w: %s carried %d %s tools and not one real input schema (%s; first: %s). "+ + "Pricing them would shrink the naive baseline and INFLATE the reported savings, so this listing is refused rather than returned", + ErrStubToolSchemas, surface, c.tools, pop.label, cause, c.example) + } + return nil +} + +// FleetFromToolsList fetches a listing and wraps its UPSTREAM tools as a +// Corpus fit to be a replay fleet. +// +// The built-ins are dropped, and that is a modelling decision worth stating: a +// fleet is the upstream catalog. Replay prices mcpproxy's own tools per cell +// from ProxyToolsForMode, the frozen corpora contain upstream tools only, and +// folding describe_tool into the fleet would both double-count it in the +// retrieve cells and move the direct cells off the basis every frozen-corpus +// figure is quoted on. (replay.go already records the reverse gap: the direct +// menu omits describe_tool, which biases the direct delta percentage slightly +// upward. Keeping the fleet upstream-only keeps that ONE stated gap rather +// than adding a second, opposite one that depends on the fleet's source.) +func FleetFromToolsList(ctx context.Context, surface, fleetID string, page ToolsListPageFunc) (*Corpus, error) { + tools, err := FetchToolsList(ctx, surface, page) + if err != nil { + return nil, err + } + upstream := make([]Tool, 0, len(tools)) + for _, t := range tools { + if t.Server == "" { + continue + } + upstream = append(upstream, t) + } + // A surface with no upstream tools is not a small fleet — it is the wrong + // surface. Under the retrieve_tools and code_execution routing modes + // tools/list returns mcpproxy's built-ins and nothing else, and reporting + // those as a fleet would price the proxy's own menu as the upstream catalog. + if len(upstream) == 0 { + return nil, fmt.Errorf("mcp tools/list %s: the listing carried %d tools but none of them is an upstream tool — "+ + "only the direct surface (%s) enumerates the fleet; the retrieve_tools and code_execution surfaces list mcpproxy's own built-ins", + surface, len(tools), EndpointDirect) + } + return &Corpus{Version: fleetID, Tools: upstream}, nil +} + +// FleetEndpointURL builds the URL a live fleet is read from. +// +// The path comes from the direct_full mode cell rather than a literal, so the +// surface a fleet is captured from stays tied to the one cell that actually +// enumerates upstream tools; if the matrix ever remounts that surface, this +// follows it. The API key rides the ?apikey= query parameter, the same surface +// mcpEndpointURL uses, and never appears in any reported field. +func FleetEndpointURL(baseURL, apiKey string) (string, error) { + cell, ok := CellByID(CellDirectFull) + if !ok { + return "", fmt.Errorf("mode cell %q is not in the matrix, so no fleet endpoint can be derived", CellDirectFull) + } + endpoint, err := cell.EndpointURL(baseURL) + if err != nil { + return "", err + } + if apiKey != "" { + endpoint += "?apikey=" + url.QueryEscape(apiKey) + } + return endpoint, nil +} + +// FleetSource supplies a replay fleet on demand. +// +// It is an interface for the reason EncodingArm is one (armrun.go): the seam +// must be satisfiable by a value a test writes inline, not only by a live +// client. It also keeps the "exactly one fleet input" rule (ResolveFleet) in +// package bench, where the fleet-is-mandatory rule already lives, instead of +// splitting the two halves of one rule across bench and the CLI. +type FleetSource interface { + // FleetID names the fleet shape every figure is quoted for (IC-004). It + // must never carry a credential: it is reported. + FleetID() string + // Fleet returns the tool definitions the menu cost is computed from. + Fleet() (*Corpus, error) +} + +// LiveFleetSource reads a fleet from a running proxy's direct MCP surface. +// +// It holds the run's context rather than taking one per call, so FleetSource +// stays a two-method seam a test can satisfy with a literal. The value is a +// one-shot adapter constructed at the call site for a single run, which is the +// case where a stored context is honest rather than a leak. +type LiveFleetSource struct { + ctx context.Context + baseURL string + apiKey string + endpoint string + fleetID string +} + +// NewLiveFleetSource builds a fleet source against a running proxy's base URL +// (e.g. "http://127.0.0.1:8080"). Endpoint resolution failures are deferred to +// Fleet(), so a mis-typed URL fails where every other fleet failure does. +func NewLiveFleetSource(ctx context.Context, baseURL, apiKey string) *LiveFleetSource { + // The fleet id names the SURFACE, never the key: it is written into the + // report. + surface := EndpointDirect + if cell, ok := CellByID(CellDirectFull); ok && cell.Endpoint != "" { + surface = cell.Endpoint + } + return &LiveFleetSource{ + ctx: ctx, + baseURL: baseURL, + apiKey: apiKey, + fleetID: "live:" + strings.TrimRight(baseURL, "/") + surface, + } +} + +// FleetID implements FleetSource. +func (s *LiveFleetSource) FleetID() string { return s.fleetID } + +// Fleet implements FleetSource: one MCP session, one paginated tools/list, and +// the stub guard. A failure here is fatal to the run by design — falling back +// to a frozen corpus would answer a different question than the one asked. +func (s *LiveFleetSource) Fleet() (*Corpus, error) { + endpoint := s.endpoint + if endpoint == "" { + resolved, err := FleetEndpointURL(s.baseURL, s.apiKey) + if err != nil { + return nil, err + } + endpoint = resolved + } + + caller, err := NewMCPRetrieveCaller(s.ctx, endpoint) + if err != nil { + return nil, err + } + defer caller.Close() + + return FleetFromToolsList(s.ctx, EndpointDirect, s.fleetID, caller.ToolsListPage) +} diff --git a/bench/mcptools_test.go b/bench/mcptools_test.go new file mode 100644 index 000000000..57945528e --- /dev/null +++ b/bench/mcptools_test.go @@ -0,0 +1,520 @@ +package bench + +// mcptools_test.go — Spec 103: the live MCP tools/list primitive. +// +// These tests hold six boundaries. Each one, crossed silently, produces a +// report whose menu cost is wrong in the direction that FLATTERS mcpproxy — +// which is the single error class this benchmark exists to prevent: +// +// 1. A listing whose input schemas are stubs is REFUSED, not returned. A +// stub-schema fleet prices every upstream definition at a fraction of its +// real size, so the naive baseline shrinks and the proxy's savings inflate. +// The Spec 102 deferred placeholder is the live instance of this hazard. +// 2. A tool that genuinely takes no parameters is NOT a stub-schema listing. +// Refusing those would make the guard useless on real fleets, so the rule +// is per-population ("nothing real anywhere"), not per-tool. +// 3. Server attribution is recovered from the "__" display name +// via the production parser, and a name that carries no prefix is a +// built-in — never a server called "". +// 4. The "[server] " description prefix the direct surface prepends is +// stripped, because the arms re-synthesize each cell's rendering from the +// canonical definition and direct_deferred prepends that prefix itself. +// 5. A replay takes exactly ONE fleet input. Two is an error, because +// silently preferring one makes the report's fleet_id a lie. +// 6. A fleet input stays MANDATORY. This primitive adds a second way to +// supply one; it does not make one optional. +// +// The transport is behind a ToolsListPageFunc seam (the bench/flipgate.go +// precedent), so every one of these runs with no proxy anywhere. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "testing" +) + +// wireEntry is a hand-built tools/list entry. It is assembled as raw JSON +// rather than through mcp.Tool on purpose: mcp-go's ToolInputSchema marshaller +// ALWAYS emits "properties" and "required", so round-tripping the Spec 102 +// placeholder {"type":"object"} through it would rewrite it as +// {"properties":{},"required":[],"type":"object"} and erase the very +// distinction these tests assert on. +type wireEntry struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + InputSchema string `json:"-"` +} + +// staticPager returns a ToolsListPageFunc that serves the given entries as a +// single unpaginated page. +func staticPager(entries ...wireEntry) ToolsListPageFunc { + return pagedPager([][]wireEntry{entries}...) +} + +// pagedPager serves one page per slice, chaining them with nextCursor so the +// pagination loop is exercised for real. +func pagedPager(pages ...[]wireEntry) ToolsListPageFunc { + byCursor := map[string]int{"": 0} + for i := range pages { + if i+1 < len(pages) { + byCursor[fmt.Sprintf("cursor-%d", i+1)] = i + 1 + } + } + return func(_ context.Context, cursor string) (json.RawMessage, error) { + idx, ok := byCursor[cursor] + if !ok { + return nil, fmt.Errorf("unknown cursor %q", cursor) + } + var parts []string + for _, e := range pages[idx] { + schema := e.InputSchema + if schema == "" { + schema = "null" + } + parts = append(parts, fmt.Sprintf(`{"name":%q,"description":%q,"inputSchema":%s}`, + e.Name, e.Description, schema)) + } + next := "" + if idx+1 < len(pages) { + next = fmt.Sprintf("cursor-%d", idx+1) + } + return json.RawMessage(fmt.Sprintf(`{"tools":[%s],"nextCursor":%q}`, + strings.Join(parts, ","), next)), nil + } +} + +const realSchema = `{"type":"object","properties":{"repo_path":{"type":"string"}},"required":["repo_path"]}` + +// paramlessSchema is what a genuinely parameter-less upstream tool serves — +// filesystem:list_allowed_directories, memory:read_graph and sqlite:list_tables +// all serve exactly these bytes on BOTH the REST and MCP surfaces. +const paramlessSchema = `{"properties":{},"required":[],"type":"object"}` + +// healthyDirectListing is the shape a full-mode /mcp/all listing has: real +// upstream schemas, "[server] " description prefixes, plus the describe_tool +// built-in the direct surface always registers alongside them. +func healthyDirectListing() ToolsListPageFunc { + return staticPager( + wireEntry{Name: "git__git_log", Description: "[git] Shows the commit logs", InputSchema: realSchema}, + wireEntry{Name: "sqlite__list_tables", Description: "[sqlite] List tables", InputSchema: paramlessSchema}, + wireEntry{Name: "describe_tool", Description: "Return full schemas", InputSchema: realSchema}, + ) +} + +// --- boundary 3 + 4: identity mapping ------------------------------------- + +func TestFetchToolsList_MapsUpstreamAndBuiltinIdentity(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp/all", healthyDirectListing()) + if err != nil { + t.Fatalf("healthy listing must be accepted: %v", err) + } + if len(tools) != 3 { + t.Fatalf("want 3 tools, got %d", len(tools)) + } + + up := tools[0] + if up.Server != "git" || up.Name != "git_log" { + t.Errorf("server attribution: want (git, git_log), got (%q, %q)", up.Server, up.Name) + } + if up.ToolID != "git:git_log" { + t.Errorf("upstream tool id must be the canonical server:tool, got %q", up.ToolID) + } + // Boundary 4: the wire prefix is stripped, so the fleet carries the same + // canonical description a frozen corpus or GET /api/v1/tools does. + if up.Description != "Shows the commit logs" { + t.Errorf("the [server] prefix must be stripped, got %q", up.Description) + } + + builtin := tools[2] + if builtin.Server != "" { + t.Errorf("a name with no %q separator is a built-in, not a server called %q", "__", builtin.Server) + } + if builtin.ToolID != "mcpproxy:describe_tool" { + t.Errorf("built-in id must match ProxyToolsForMode's convention, got %q", builtin.ToolID) + } + if builtin.Description != "Return full schemas" { + t.Errorf("a built-in description carries no prefix to strip, got %q", builtin.Description) + } +} + +// The prefix is stripped only when it names THIS tool's server. A description +// that merely happens to open with a bracket keeps every byte. +func TestFetchToolsList_StripsOnlyItsOwnServerPrefix(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "[notes] see [git] elsewhere", InputSchema: realSchema}, + )) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tools[0].Description != "[notes] see [git] elsewhere" { + t.Errorf("a foreign bracket must survive verbatim, got %q", tools[0].Description) + } +} + +// Schemas are canonicalized at the ingestion boundary, exactly as +// LiveClient.FetchUpstreamTools does, so the bytes the tokenizer counts match +// what the arm renderers produce. +func TestFetchToolsList_CanonicalizesSchemasAtTheBoundary(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "d", InputSchema: `{ "required" : ["b"] , "type":"object", "properties":{"b":{"type":"string"}} }`}, + )) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := `{"properties":{"b":{"type":"string"}},"required":["b"],"type":"object"}` + if string(tools[0].Schema) != want { + t.Errorf("schema must be canonical:\n want %s\n got %s", want, tools[0].Schema) + } +} + +// --- boundary 1: the stub guard ------------------------------------------- + +// The Spec 102 deferred direct surface serves {"type":"object"} for EVERY +// upstream tool while describe_tool keeps a real schema. A guard that only +// asked "does anything here have a schema" would wave this through and +// under-count the direct menu by ~39%. +func TestFetchToolsList_RefusesDeferredPlaceholderSchemas(t *testing.T) { + _, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "[git] Shows the commit logs\ngit_log(repo_path*:str)", InputSchema: `{"type":"object"}`}, + wireEntry{Name: "sqlite__list_tables", Description: "[sqlite] List tables\nlist_tables()", InputSchema: `{"type":"object"}`}, + wireEntry{Name: "describe_tool", Description: "Return full schemas", InputSchema: realSchema}, + )) + if err == nil { + t.Fatal("a schema-deferred listing must be REFUSED: every upstream definition would be priced at a fraction of its real size") + } + if !errors.Is(err, ErrStubToolSchemas) { + t.Fatalf("want ErrStubToolSchemas, got %v", err) + } + // The error must be actionable: name the surface, and name the config that + // produced the placeholder. + for _, want := range []string{"/mcp/all", "direct_tool_response_mode", "deferred", "upstream"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("stub error must mention %q; got: %v", want, err) + } + } +} + +// The supervisor StateView stub (MCP-3167) is the other systematic source. It +// is fixed on the REST side today, but the guard is the thing that keeps a +// regression from silently re-inflating the headline. +func TestFetchToolsList_RefusesSupervisorStubSchemas(t *testing.T) { + _, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "[git] Shows the commit logs", InputSchema: `{"type":"object","properties":{}}`}, + wireEntry{Name: "fs__read_file", Description: "[fs] Read a file", InputSchema: `{"type":"object","properties":{}}`}, + )) + if !errors.Is(err, ErrStubToolSchemas) { + t.Fatalf("want ErrStubToolSchemas for an all-stub upstream population, got %v", err) + } +} + +// A listing that carries no schemas at all is the same failure with the field +// missing rather than emptied. +func TestFetchToolsList_RefusesSchemalessListing(t *testing.T) { + _, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "[git] Shows the commit logs"}, + wireEntry{Name: "fs__read_file", Description: "[fs] Read a file"}, + )) + if !errors.Is(err, ErrStubToolSchemas) { + t.Fatalf("want ErrStubToolSchemas for a schema-less listing, got %v", err) + } +} + +// The built-in population is guarded independently, so a stubbed /mcp listing +// is refused even though no upstream tool appears on that surface at all. +func TestFetchToolsList_GuardsTheBuiltinPopulationSeparately(t *testing.T) { + _, err := FetchToolsList(context.Background(), "/mcp", staticPager( + wireEntry{Name: "retrieve_tools", Description: "Search", InputSchema: `{"type":"object","properties":{}}`}, + wireEntry{Name: "list_registries", Description: "List", InputSchema: paramlessSchema}, + )) + if !errors.Is(err, ErrStubToolSchemas) { + t.Fatalf("want ErrStubToolSchemas for an all-stub built-in listing, got %v", err) + } + if !strings.Contains(err.Error(), "built-in") { + t.Errorf("the error must name the population it found empty; got: %v", err) + } +} + +// --- boundary 2: parameter-less tools are not stubs ------------------------ + +func TestFetchToolsList_ParameterlessToolsAreNotAStubListing(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp/all", healthyDirectListing()) + if err != nil { + t.Fatalf("3 real + 1 parameter-less must be accepted, got: %v", err) + } + // The parameter-less schema is preserved rather than dropped: those bytes + // are on the wire and the agent pays for them. + if len(tools[1].Schema) == 0 { + t.Error("a parameter-less tool's schema is real wire bytes and must be kept") + } +} + +// A /mcp listing of healthy built-ins passes with no upstream population at +// all — an absent population is not an empty one. +func TestFetchToolsList_BuiltinOnlyListingIsAccepted(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp", staticPager( + wireEntry{Name: "retrieve_tools", Description: "Search", InputSchema: realSchema}, + wireEntry{Name: "list_registries", Description: "List", InputSchema: paramlessSchema}, + )) + if err != nil { + t.Fatalf("a built-ins-only surface must be accepted: %v", err) + } + if len(tools) != 2 { + t.Fatalf("want 2 built-ins, got %d", len(tools)) + } +} + +// --- listing hygiene ------------------------------------------------------- + +func TestFetchToolsList_FollowsPagination(t *testing.T) { + tools, err := FetchToolsList(context.Background(), "/mcp/all", pagedPager( + []wireEntry{{Name: "git__git_log", Description: "[git] a", InputSchema: realSchema}}, + []wireEntry{{Name: "fs__read_file", Description: "[fs] b", InputSchema: realSchema}}, + )) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(tools) != 2 { + t.Fatalf("nextCursor must be followed; want 2 tools, got %d", len(tools)) + } + if tools[0].ToolID != "git:git_log" || tools[1].ToolID != "fs:read_file" { + t.Errorf("page order must be preserved, got %q then %q", tools[0].ToolID, tools[1].ToolID) + } +} + +// A duplicate display name would be counted twice in the menu. The direct +// catalog withholds colliding names server-side, so a duplicate reaching bench +// means something upstream of it is wrong — refuse rather than double-count. +func TestFetchToolsList_RefusesDuplicateToolIDs(t *testing.T) { + _, err := FetchToolsList(context.Background(), "/mcp/all", staticPager( + wireEntry{Name: "git__git_log", Description: "[git] a", InputSchema: realSchema}, + wireEntry{Name: "git__git_log", Description: "[git] a", InputSchema: realSchema}, + )) + if err == nil || !strings.Contains(err.Error(), "git:git_log") { + t.Fatalf("a duplicate tool id must be refused and named, got %v", err) + } +} + +func TestFetchToolsList_RefusesEmptyListing(t *testing.T) { + if _, err := FetchToolsList(context.Background(), "/mcp/all", staticPager()); err == nil { + t.Fatal("an empty listing must be an error: a zero-tool fleet would price a zero menu as a measurement") + } +} + +func TestFetchToolsList_PropagatesTransportFailure(t *testing.T) { + boom := errors.New("connection refused") + _, err := FetchToolsList(context.Background(), "/mcp/all", + func(context.Context, string) (json.RawMessage, error) { return nil, boom }) + if !errors.Is(err, boom) { + t.Fatalf("transport failure must propagate, got %v", err) + } +} + +// --- fleet construction ---------------------------------------------------- + +// A FLEET is the upstream catalog. mcpproxy's own built-ins are priced per +// cell from ProxyToolsForMode, so folding them into the fleet would count them +// twice in the retrieve cells and move the direct cells off the basis every +// frozen-corpus figure is quoted on. +func TestFleetFromToolsList_DropsBuiltins(t *testing.T) { + corpus, err := FleetFromToolsList(context.Background(), "/mcp/all", "live:test", healthyDirectListing()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(corpus.Tools) != 2 { + t.Fatalf("describe_tool must not enter the fleet; want 2 upstream tools, got %d", len(corpus.Tools)) + } + for _, tl := range corpus.Tools { + if tl.Server == "" { + t.Errorf("a fleet tool must carry a server, got %+v", tl) + } + } + if corpus.Version != "live:test" { + t.Errorf("fleet version must be the fleet id, got %q", corpus.Version) + } +} + +// Pointing the fleet fetch at a retrieve_tools or code_execution surface +// returns only built-ins. That is not a small fleet — it is the wrong surface, +// and reporting it as a fleet would price mcpproxy's own menu as the upstream +// catalog. +func TestFleetFromToolsList_RefusesASurfaceWithNoUpstreamTools(t *testing.T) { + _, err := FleetFromToolsList(context.Background(), "/mcp", "live:test", staticPager( + wireEntry{Name: "retrieve_tools", Description: "Search", InputSchema: realSchema}, + )) + if err == nil { + t.Fatal("a built-ins-only surface is not a fleet and must be refused") + } + if !strings.Contains(err.Error(), EndpointDirect) { + t.Errorf("the error must point at the surface that does enumerate the fleet (%s); got: %v", EndpointDirect, err) + } +} + +// --- boundaries 5 + 6: one fleet, and never zero --------------------------- + +// fakeFleetSource is a structural FleetSource, the way replay_test.go's arms +// are structural EncodingArms. +type fakeFleetSource struct { + id string + corpus *Corpus + err error +} + +func (f *fakeFleetSource) FleetID() string { return f.id } +func (f *fakeFleetSource) Fleet() (*Corpus, error) { return f.corpus, f.err } + +func liveSource() *fakeFleetSource { + return &fakeFleetSource{ + id: "live:127.0.0.1:18421/mcp/all", + corpus: &Corpus{Version: "live:127.0.0.1:18421/mcp/all", Tools: []Tool{ + {ToolID: "git:git_log", Server: "git", Name: "git_log", Description: "d", Schema: json.RawMessage(realSchema)}, + }}, + } +} + +// Boundary 5 — two fleets is ambiguous. Preferring one silently would make the +// report's fleet_id a lie about which definitions every figure was priced over. +func TestResolveFleet_TwoFleetInputsIsAnError(t *testing.T) { + _, _, err := ResolveFleet(runnerCorpus(), "corpus_test@1", liveSource()) + if err == nil { + t.Fatal("a frozen corpus AND a live proxy must be refused") + } + if !errors.Is(err, ErrReplayFleetAmbiguous) { + t.Fatalf("want ErrReplayFleetAmbiguous, got %v", err) + } +} + +// Boundary 6 — the live source ADDS a way to supply a fleet; it does not make +// one optional. +func TestResolveFleet_NoFleetInputIsStillRequired(t *testing.T) { + if _, _, err := ResolveFleet(nil, "", nil); !errors.Is(err, ErrReplayFleetRequired) { + t.Fatalf("want ErrReplayFleetRequired, got %v", err) + } +} + +func TestResolveFleet_LiveSourceSuppliesFleetAndID(t *testing.T) { + fleet, id, err := ResolveFleet(nil, "", liveSource()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(fleet.Tools) != 1 || fleet.Tools[0].ToolID != "git:git_log" { + t.Fatalf("live fleet not passed through: %+v", fleet) + } + if id != "live:127.0.0.1:18421/mcp/all" { + t.Errorf("fleet id must name the live surface, got %q", id) + } +} + +// A live fetch that fails must fail the run. Falling back to a frozen corpus +// would answer a different question than the one that was asked. +func TestResolveFleet_LiveFetchFailureIsFatal(t *testing.T) { + boom := errors.New("connection refused") + if _, _, err := ResolveFleet(nil, "", &fakeFleetSource{id: "live:x", err: boom}); !errors.Is(err, boom) { + t.Fatalf("a failed live fleet fetch must propagate, got %v", err) + } +} + +// An explicit frozen corpus keeps its own id, unchanged by this feature. +func TestResolveFleet_FrozenCorpusUnchanged(t *testing.T) { + fleet, id, err := ResolveFleet(runnerCorpus(), "corpus_test@1", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fleet == nil || id != "corpus_test@1" { + t.Fatalf("frozen path regressed: fleet=%v id=%q", fleet, id) + } +} + +// --- endpoint construction ------------------------------------------------- + +// The fleet endpoint is derived from the direct_full mode cell rather than a +// hardcoded path, so the surface a fleet is read from stays tied to the one +// cell that enumerates it. +func TestFleetEndpointURL_UsesTheDirectCellAndHidesTheKey(t *testing.T) { + got, err := FleetEndpointURL("http://127.0.0.1:18421/", "s3cret") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.HasPrefix(got, "http://127.0.0.1:18421"+EndpointDirect) { + t.Errorf("fleet endpoint must be the direct surface, got %q", got) + } + + // The key belongs on the wire, never in a report field. + src := NewLiveFleetSource(context.Background(), "http://127.0.0.1:18421", "s3cret") + if strings.Contains(src.FleetID(), "s3cret") { + t.Errorf("the fleet id must never carry the API key, got %q", src.FleetID()) + } + if !strings.Contains(src.FleetID(), EndpointDirect) { + t.Errorf("the fleet id must name the surface it was read from, got %q", src.FleetID()) + } +} + +// D6 — the API key must actually reach the URL. +// +// The previous coverage asserted only that the key is ABSENT from FleetID and +// that the path has the direct-cell prefix, so replacing the whole +// key-appending branch with `if false` left the suite green. An unauthenticated +// fetch against a proxy that requires a key fails at connect time rather than +// silently, but the assertion should hold the behaviour, not the accident. +func TestFleetEndpointURL_CarriesTheAPIKey(t *testing.T) { + got, err := FleetEndpointURL("http://127.0.0.1:18421", "s3cr3t key/with+chars") + if err != nil { + t.Fatalf("FleetEndpointURL: %v", err) + } + if !strings.Contains(got, "apikey=") { + t.Fatalf("the key must reach the query string, got %q", got) + } + if strings.Contains(got, "s3cr3t key/with+chars") { + t.Errorf("the key must be URL-escaped, not interpolated raw: %q", got) + } + if !strings.Contains(got, url.QueryEscape("s3cr3t key/with+chars")) { + t.Errorf("the escaped key must appear verbatim: %q", got) + } + + // And no key means no query string at all — an empty apikey= would be a + // different request from an unauthenticated one. + bare, err := FleetEndpointURL("http://127.0.0.1:18421", "") + if err != nil { + t.Fatalf("FleetEndpointURL(bare): %v", err) + } + if strings.Contains(bare, "apikey") { + t.Errorf("an empty key must not produce an apikey parameter, got %q", bare) + } +} + +// D7 — the repeated-cursor guard must be exercised. +// +// Disabling it left the suite green, yet a server that repeats a cursor would +// duplicate the whole catalog into the fleet and double its menu cost — an +// error that shows up as an implausibly large baseline rather than as a crash. +func TestFetchToolsList_RefusesARepeatedCursor(t *testing.T) { + calls := 0 + looping := func(_ context.Context, _ string) (json.RawMessage, error) { + calls++ + // Always hands back the SAME nextCursor. + return json.RawMessage(`{"tools":[{"name":"git__git_log","description":"d",` + + `"inputSchema":{"type":"object","properties":{"repo_path":{"type":"string"}}}}],` + + `"nextCursor":"same-cursor"}`), nil + } + + _, err := FetchToolsList(context.Background(), "/mcp/all", looping) + if err == nil { + t.Fatal("a repeated cursor must be refused — pagination would loop and duplicate the catalog") + } + if !strings.Contains(err.Error(), "repeated cursor") { + t.Errorf("the error must name the cause; got %v", err) + } + if calls > 3 { + t.Errorf("the guard should trip on the second sighting, not after %d pages", calls) + } +} + +// D7 — the nil-transport guard must be exercised too. +func TestFetchToolsList_NilTransportIsAnError(t *testing.T) { + if _, err := FetchToolsList(context.Background(), "/mcp/all", nil); err == nil { + t.Error("a nil page function must be an error, not a nil fleet") + } +} diff --git a/bench/payloaddecomp.go b/bench/payloaddecomp.go new file mode 100644 index 000000000..e0aa8dd7b --- /dev/null +++ b/bench/payloaddecomp.go @@ -0,0 +1,312 @@ +package bench + +import ( + "encoding/json" + "fmt" +) + +// Spec 103 US4 — payload decomposition. +// +// Spec 102 projected ~70% savings from deferring input schemas and measured +// 29.7% on the frozen 45-tool corpus, concluding that names, descriptions and +// annotations carry most of the payload rather than schemas. That conclusion +// was drawn from two frozen corpora and never checked against another shape, +// and it is the single claim every reader of the benchmark will want tested. +// +// This file tests it the only way that can be honest: attribute the payload of +// a real corpus to its components and recompute the achievable ceiling FOR THAT +// CORPUS. +// +// Two measurement hazards shape the implementation, and both would produce a +// plausible-looking wrong answer: +// +// 1. TOKENS ARE NOT ADDITIVE ACROSS CONCATENATION. BPE merges across component +// boundaries, so Count(name) + Count(description) + Count(schema) does not +// equal Count(name + "\n" + description + "\n" + schema). Counting each +// component separately yields shares that never sum to the payload and that +// misattribute every boundary. So the payload is tokenized ONCE and each +// token is attributed to a component by byte offset, reusing the span +// partition AttributeTokens already implements for response cost. +// +// 2. THE CORPUS CARRIES NO ANNOTATIONS. bench.Tool is {ToolID, Server, Name, +// Description, Schema}. Spec 102 measured the production WIRE payload, +// which includes mcp-go's annotation defaults; the frozen corpora do not. +// These are different payloads, and a decomposition that silently reported +// a three-way split as though it settled a claim about four components +// would read as a confirmation while measuring something else. The +// Limitation field says so, in the report, next to the verdict. + +// PayloadComponent labels the parts of a rendered tool definition. +const ( + PayloadComponentName = "name" + PayloadComponentDescription = "description" + PayloadComponentSchema = "schema" + // PayloadComponentStructural is the rendering's own overhead — the + // separators between components. Small, but it belongs to some component + // or the shares do not partition the payload, and folding it into a + // content component would overstate that component. + PayloadComponentStructural = "structural" +) + +// PayloadDecomposition attributes one corpus's rendered payload to its +// components, and states what that corpus shape can and cannot settle. +type PayloadDecomposition struct { + FleetID string `json:"fleet_id"` + ToolCount int `json:"tool_count"` + TotalTokens int `json:"total_tokens"` + // ToolsWithSchema is how many of ToolCount actually carry an input schema. + // Load-bearing: a corpus with none cannot measure a schema share at all, + // and a 0% result there means "nothing to defer was present", not "schemas + // are cheap". Reporting the second from the first is the failure this + // field exists to make impossible. + ToolsWithSchema int `json:"tools_with_schema"` + + NameTokens int `json:"name_tokens"` + DescriptionTokens int `json:"description_tokens"` + SchemaTokens int `json:"schema_tokens"` + StructuralTokens int `json:"structural_tokens"` + + NamePct float64 `json:"name_pct"` + DescriptionPct float64 `json:"description_pct"` + SchemaPct float64 `json:"schema_pct"` + StructuralPct float64 `json:"structural_pct"` + + // SchemaCeilingPct is the most any schema-deferring design could ever save + // on THIS corpus: the schema share, since deleting the schema entirely is + // the upper bound of deferring it. Recomputed per corpus — carrying a + // ceiling forward is the error spec 102 made. + SchemaCeilingPct float64 `json:"schema_ceiling_pct"` + + // Spec102Verdict is an explicit confirmed/corrected statement rather than + // a number the reader has to interpret. + Spec102Verdict string `json:"spec_102_verdict"` + + // Limitation names what this corpus shape cannot settle. Always populated: + // a decomposition with no stated limits is claiming there are none. + Limitation string `json:"limitation"` + + Provenance string `json:"provenance"` +} + +// DecomposePayload attributes the canonical rendering of every tool in the +// corpus to name, description, schema and structural overhead. +// +// The rendering matches the baseline arm's canonical form — +// name + "\n" + description + "\n" + canonical(schema) — because that is what +// the savings figures are measured against. Decomposing a different rendering +// would produce shares that do not explain those figures. +func DecomposePayload(tk *Tokenizer, corpus *Corpus, fleetID string) (*PayloadDecomposition, error) { + if tk == nil { + return nil, fmt.Errorf("payload decomposition: nil tokenizer") + } + if corpus == nil || len(corpus.Tools) == 0 { + // Never a zero-filled result: zeros render as "measured, and every + // component costs nothing", which is worse than an error. + return nil, fmt.Errorf("payload decomposition: corpus %q has no tools", fleetID) + } + + d := &PayloadDecomposition{ + FleetID: fleetID, + ToolCount: len(corpus.Tools), + Provenance: ProvenanceMeasured, + } + + for _, tl := range corpus.Tools { + if len(tl.Schema) > 0 { + d.ToolsWithSchema++ + } + text, spans, err := partitionToolDefinition(tl) + if err != nil { + return nil, fmt.Errorf("payload decomposition: tool %s: %w", tl.ToolID, err) + } + total, components, err := AttributeTokens(tk, text, spans) + if err != nil { + return nil, fmt.Errorf("payload decomposition: tool %s: %w", tl.ToolID, err) + } + d.TotalTokens += total + d.NameTokens += components[PayloadComponentName] + d.DescriptionTokens += components[PayloadComponentDescription] + d.SchemaTokens += components[PayloadComponentSchema] + d.StructuralTokens += components[PayloadComponentStructural] + } + + if d.TotalTokens > 0 { + pct := func(n int) float64 { return float64(n) / float64(d.TotalTokens) * 100 } + d.NamePct = pct(d.NameTokens) + d.DescriptionPct = pct(d.DescriptionTokens) + d.SchemaPct = pct(d.SchemaTokens) + d.StructuralPct = pct(d.StructuralTokens) + d.SchemaCeilingPct = d.SchemaPct + } + + d.Spec102Verdict = spec102Verdict(d) + d.Limitation = "This decomposition covers the corpus rendering only: name, description and " + + "input schema. bench.Tool carries no annotations field, while spec 102 measured the " + + "production wire payload including mcp-go's annotation defaults. The annotations third " + + "of its conclusion is therefore OUT OF REACH of any corpus-based decomposition and is " + + "neither confirmed nor refuted here." + + return d, nil +} + +// spec102Verdict states what THIS CORPUS RENDERING shows, and is careful not to +// claim more than that. +// +// Two ways a naive verdict goes wrong here, both found by running the first +// version against the real corpora: +// +// 1. A CORPUS WITH NO SCHEMAS yields a 0% schema share, and a verdict phrased +// as "deferring schemas cannot exceed 0%" reads as a resounding +// confirmation of spec 102. It is nothing of the kind — it measures a +// corpus that does not contain the thing under test. The committed +// 527-tool snapshot is exactly this case: all 527 tools, zero schemas. +// +// 2. IT IS A DIFFERENT PAYLOAD FROM THE ONE SPEC 102 MEASURED. Spec 102 +// measured the production wire payload — mcp.Tool marshalling, the +// "[server] " prefix, mcp-go's annotation defaults, the raw-schema +// placeholder. This decomposition covers the corpus rendering, which has no +// annotations and no wire framing. A confident CONFIRMED/CORRECTED across +// that gap would be adjudicating a claim on evidence about something else. +// +// So the verdict describes the corpus and explicitly declines to settle spec +// 102's figure. The decomposition is still the useful thing — it says where the +// payload actually goes on a given fleet — it just is not a verdict on a +// measurement taken elsewhere. +func spec102Verdict(d *PayloadDecomposition) string { + prose := d.NamePct + d.DescriptionPct + const scope = " This describes the CORPUS RENDERING (name, description, schema). Spec 102's " + + "29.7% figure was measured on the production WIRE payload, which additionally carries " + + "annotations and wire framing, so this neither confirms nor corrects it." + + if d.ToolsWithSchema == 0 { + return fmt.Sprintf("INAPPLICABLE: none of the %d tools in this corpus carries an input "+ + "schema, so there is no schema share to measure and no ceiling to compute. A 0%% "+ + "result here means the corpus lacks the thing under test, NOT that schemas are "+ + "cheap.%s", d.ToolCount, scope) + } + + coverage := float64(d.ToolsWithSchema) / float64(d.ToolCount) * 100 + partial := "" + if d.ToolsWithSchema < d.ToolCount { + partial = fmt.Sprintf(" Note only %d of %d tools (%.0f%%) carry a schema, so the share is "+ + "diluted by the schema-less remainder.", d.ToolsWithSchema, d.ToolCount, coverage) + } + + switch { + case d.SchemaPct >= 50: + return fmt.Sprintf("SCHEMA-DOMINANT on this corpus: schemas are %.1f%% of the payload "+ + "against %.1f%% for names and descriptions, so deferring them could save up to "+ + "%.1f%% here.%s%s", d.SchemaPct, prose, d.SchemaCeilingPct, partial, scope) + case prose > d.SchemaPct: + return fmt.Sprintf("PROSE-DOMINANT on this corpus: names and descriptions are %.1f%% "+ + "against %.1f%% for schemas, so deferring schemas cannot exceed %.1f%% here however "+ + "it is implemented.%s%s", prose, d.SchemaPct, d.SchemaCeilingPct, partial, scope) + default: + return fmt.Sprintf("EVENLY SPLIT on this corpus: schemas %.1f%%, names plus descriptions "+ + "%.1f%%. Ceiling %.1f%%.%s%s", d.SchemaPct, prose, d.SchemaCeilingPct, partial, scope) + } +} + +// partitionToolDefinition builds the canonical rendering of one tool together +// with a span partition covering it exactly. +// +// The spans must be contiguous and cover [0, len(text)) — AttributeTokens +// enforces that — which is why the separators get their own structural spans +// rather than being folded into a neighbour. +func partitionToolDefinition(tl Tool) (string, []Span, error) { + var ( + text string + spans []Span + ) + + add := func(label, s string) { + if s == "" { + return + } + start := len(text) + text += s + spans = append(spans, Span{Label: label, Start: start, End: len(text)}) + } + + add(PayloadComponentName, tl.Name) + add(PayloadComponentStructural, "\n") + add(PayloadComponentDescription, tl.Description) + + if len(tl.Schema) > 0 { + canon, err := canonicalSchemaJSON(tl.Schema) + if err != nil { + return "", nil, err + } + add(PayloadComponentStructural, "\n") + add(PayloadComponentSchema, canon) + } + + if text == "" { + return "", nil, fmt.Errorf("tool renders to an empty definition") + } + return text, spans, nil +} + +// canonicalSchemaJSON matches the canonicalisation the baseline arm applies, so +// the decomposition explains the same payload the savings figures are measured +// against rather than a re-marshalled approximation of it. +// +// bench cannot import bench/arms (arms import bench), so the canonical form is +// reproduced here; the drift risk is bounded because both are "unmarshal into +// interface{}, re-marshal with sorted keys", which encoding/json does by +// construction for maps. +func canonicalSchemaJSON(raw json.RawMessage) (string, error) { + var v interface{} + if err := json.Unmarshal(raw, &v); err != nil { + return "", fmt.Errorf("schema is not valid JSON: %w", err) + } + out, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("schema could not be re-marshalled: %w", err) + } + return string(out), nil +} + +// PayloadDecompositionBlockFor builds the report block from one decomposition +// per fleet shape (T053). +// +// FR-024 asks for at least two shapes, because the whole point is to show how +// the attribution MOVES with fleet size rather than to publish a single-corpus +// projection — which is the shape of mistake spec 102 made. Fewer than two is +// therefore an error rather than a smaller block. +func PayloadDecompositionBlockFor(ds []*PayloadDecomposition) (*PayloadDecompositionBlock, error) { + if len(ds) < 2 { + return nil, fmt.Errorf("payload decomposition: %d fleet shape(s); FR-024 requires at "+ + "least two so the attribution can be seen to move with fleet size", len(ds)) + } + + block := &PayloadDecompositionBlock{ + AccountingSource: AccountingSource{ + Kind: AccountingKindTokenizer, + Identity: DefaultEncoding, + }, + } + + for _, d := range ds { + if d == nil { + return nil, fmt.Errorf("payload decomposition: nil shape") + } + block.Shapes = append(block.Shapes, PayloadDecompositionRow{ + FleetShape: FleetShape{ + ID: d.FleetID, + ToolCount: d.ToolCount, + }, + Provenance: d.Provenance, + ShareNamesPct: d.NamePct, + ShareDescriptionsPct: d.DescriptionPct, + // Deliberately nil: see the field's doc comment. The corpora carry + // no annotations, and a zero here would claim a measurement that + // was never taken. + ShareAnnotationsPct: nil, + ShareSchemasPct: d.SchemaPct, + AchievableCeilingPct: d.SchemaCeilingPct, + Spec102Verdict: d.Spec102Verdict, + }) + } + return block, nil +} diff --git a/bench/payloaddecomp_test.go b/bench/payloaddecomp_test.go new file mode 100644 index 000000000..37075a308 --- /dev/null +++ b/bench/payloaddecomp_test.go @@ -0,0 +1,271 @@ +package bench + +import ( + "encoding/json" + "strings" + "testing" +) + +// decompCorpus is a small schema-bearing corpus. The schemas are deliberately +// larger than the descriptions on one tool and smaller on another, so a test +// that only ever sees one shape dominating cannot pass by accident. +func decompCorpus() *Corpus { + return &Corpus{ + Version: "decomp_test@1", + Tools: []Tool{ + { + ToolID: "srv:big_schema", + Server: "srv", + Name: "big_schema", + Description: "Short.", + Schema: json.RawMessage(`{"type":"object","properties":{` + + `"alpha":{"type":"string","description":"the first parameter of several"},` + + `"beta":{"type":"integer","description":"the second parameter of several"},` + + `"gamma":{"type":"boolean","description":"the third parameter of several"}},` + + `"required":["alpha","beta"]}`), + }, + { + ToolID: "srv:big_description", + Server: "srv", + Name: "big_description", + Description: "A deliberately long description that runs well past the length of " + + "this tool's input schema, so the decomposition has at least one tool where " + + "prose rather than structure dominates the payload.", + Schema: json.RawMessage(`{"type":"object"}`), + }, + { + ToolID: "srv:no_schema", + Server: "srv", + Name: "no_schema", + Description: "Carries no schema at all.", + }, + }, + } +} + +// T050 — the shares must PARTITION the payload, not merely approximate it. +// +// This is the whole reason the decomposition attributes token spans over a +// single tokenization instead of counting each component separately. BPE merges +// across boundaries, so Count(name) + Count(description) + Count(schema) does +// NOT equal Count(name + description + schema). Independent counts would +// produce shares that look plausible, never sum, and quietly misattribute the +// bytes at every component boundary. +func TestPayloadDecomposition_SharesPartitionThePayload(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + + sum := d.NameTokens + d.DescriptionTokens + d.SchemaTokens + d.StructuralTokens + if sum != d.TotalTokens { + t.Errorf("shares must sum to the payload exactly: %d + %d + %d + %d = %d, want %d", + d.NameTokens, d.DescriptionTokens, d.SchemaTokens, d.StructuralTokens, sum, d.TotalTokens) + } + if d.TotalTokens == 0 { + t.Fatal("a non-empty corpus must have a non-zero payload") + } + for _, c := range []struct { + label string + n int + }{ + {"name", d.NameTokens}, + {"description", d.DescriptionTokens}, + {"schema", d.SchemaTokens}, + } { + if c.n <= 0 { + t.Errorf("%s share is %d; this corpus has all three, so a zero means the "+ + "attribution collapsed rather than measured", c.label, c.n) + } + } +} + +// T051 — the ceiling is a property of the corpus and must be recomputed for +// each one. +// +// Carrying a ceiling forward is precisely the error spec 102 made: it projected +// ~70% from an assumption about one corpus shape and measured 29.7%. A +// decomposition that reused a previous corpus's ceiling would repeat it while +// looking like a measurement. +func TestPayloadDecomposition_CeilingIsPerCorpus(t *testing.T) { + tk := runnerTokenizer(t) + + schemaHeavy := decompCorpus() + proseHeavy := &Corpus{ + Version: "prose@1", + Tools: []Tool{{ + ToolID: "srv:prose", Server: "srv", Name: "prose", + Description: strings.Repeat("a great deal of description text and nothing else. ", 40), + Schema: json.RawMessage(`{"type":"object"}`), + }}, + } + + a, err := DecomposePayload(tk, schemaHeavy, "a@1") + if err != nil { + t.Fatalf("DecomposePayload(schemaHeavy): %v", err) + } + b, err := DecomposePayload(tk, proseHeavy, "b@1") + if err != nil { + t.Fatalf("DecomposePayload(proseHeavy): %v", err) + } + + if a.SchemaCeilingPct == b.SchemaCeilingPct { + t.Errorf("two corpora with opposite shapes produced an identical ceiling (%.2f%%); "+ + "the ceiling must be recomputed per corpus, never carried forward", a.SchemaCeilingPct) + } + if b.SchemaCeilingPct >= a.SchemaCeilingPct { + t.Errorf("the prose-heavy corpus must have a LOWER schema ceiling than the "+ + "schema-heavy one: prose=%.2f%% schema=%.2f%%", b.SchemaCeilingPct, a.SchemaCeilingPct) + } + if b.SchemaCeilingPct < 0 || a.SchemaCeilingPct > 100 { + t.Errorf("ceiling out of range: a=%.2f b=%.2f", a.SchemaCeilingPct, b.SchemaCeilingPct) + } +} + +// The block must state what this corpus shape CANNOT settle. +// +// Spec 102 concluded that names, descriptions AND ANNOTATIONS dominate the +// payload. bench.Tool carries no annotations field, so a corpus-based +// decomposition cannot confirm or refute the annotations third of that claim. +// Reporting a verdict without saying so would look like a confirmation and +// would not be one. +func TestPayloadDecomposition_DeclaresTheAnnotationsLimit(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if !strings.Contains(strings.ToLower(d.Limitation), "annotation") { + t.Errorf("the decomposition must state that annotations are absent from the corpus "+ + "shape and therefore out of its reach; got %q", d.Limitation) + } + if d.Spec102Verdict == "" { + t.Error("the block must carry an explicit spec-102 verdict, not leave it implied") + } +} + +// An empty corpus is an error, not a zero-filled decomposition. +// +// Zeros would render as "measured, and everything costs nothing", which is the +// same silent-zero failure the exclusion accounting exists to prevent. +func TestPayloadDecomposition_EmptyCorpusIsAnError(t *testing.T) { + tk := runnerTokenizer(t) + + if _, err := DecomposePayload(tk, &Corpus{Version: "empty@1"}, "empty@1"); err == nil { + t.Error("an empty corpus must be an error, never a zero-filled decomposition") + } +} + +// A corpus with no schemas cannot measure a schema share, and must say so +// rather than reporting a confident 0%. +// +// Found by running the first version against the committed 527-tool snapshot: +// all 527 tools, zero schemas. The verdict read "deferring schemas cannot +// exceed 0.0% however it is implemented" — which sounds like a decisive result +// and is really a corpus that does not contain the thing under test. +func TestPayloadDecomposition_SchemalessCorpusIsInapplicable(t *testing.T) { + tk := runnerTokenizer(t) + + schemaless := &Corpus{ + Version: "schemaless@1", + Tools: []Tool{ + {ToolID: "s:a", Server: "s", Name: "a", Description: "no schema here"}, + {ToolID: "s:b", Server: "s", Name: "b", Description: "nor here"}, + }, + } + + d, err := DecomposePayload(tk, schemaless, "schemaless@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if d.ToolsWithSchema != 0 { + t.Fatalf("fixture should carry no schemas, got %d", d.ToolsWithSchema) + } + if !strings.Contains(d.Spec102Verdict, "INAPPLICABLE") { + t.Errorf("a schema-less corpus must yield an INAPPLICABLE verdict, not a confident "+ + "0%% result; got %q", d.Spec102Verdict) + } +} + +// The verdict must never claim to confirm or correct spec 102. +// +// Spec 102 measured the production wire payload — annotations and framing +// included. This decomposition covers the corpus rendering. Adjudicating that +// claim across the gap would be answering with evidence about something else, +// which is precisely what the Limitation field warns of; the verdict has to +// respect its own caveat. +func TestPayloadDecomposition_VerdictDeclinesToSettleSpec102(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "decomp_test@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + v := d.Spec102Verdict + for _, forbidden := range []string{"CONFIRMED", "CORRECTED"} { + if strings.Contains(v, forbidden) { + t.Errorf("the verdict must not claim to %s spec 102 from corpus evidence — it is a "+ + "different payload; got %q", forbidden, v) + } + } + if !strings.Contains(v, "WIRE payload") { + t.Errorf("the verdict must name the payload gap it is declining to cross; got %q", v) + } +} + +// T053 — the block must carry a populated accounting source, and must report +// the annotation share as NULL rather than zero. +// +// Zero would claim a measurement that was never taken. The corpora carry no +// annotations field at all, so "not measured" is the only truthful value, and +// the difference between null and 0 is the difference between an honest gap and +// a fabricated finding. +func TestPayloadDecompositionBlock_AnnotationShareIsNullNotZero(t *testing.T) { + tk := runnerTokenizer(t) + + a, err := DecomposePayload(tk, decompCorpus(), "a@1") + if err != nil { + t.Fatalf("DecomposePayload(a): %v", err) + } + b, err := DecomposePayload(tk, decompCorpus(), "b@1") + if err != nil { + t.Fatalf("DecomposePayload(b): %v", err) + } + + block, err := PayloadDecompositionBlockFor([]*PayloadDecomposition{a, b}) + if err != nil { + t.Fatalf("PayloadDecompositionBlockFor: %v", err) + } + if block.AccountingSource.IsZero() { + t.Error("the block must name a populated accounting source") + } + for i, row := range block.Shapes { + if row.ShareAnnotationsPct != nil { + t.Errorf("shape %d: the annotation share must be null (not measurable from a "+ + "corpus that carries no annotations), got %v", i, *row.ShareAnnotationsPct) + } + if row.Provenance != ProvenanceMeasured { + t.Errorf("shape %d: provenance %q", i, row.Provenance) + } + } +} + +// FR-024 wants at least two shapes, so a single-corpus block is an error. +// +// One shape is exactly the evidence base spec 102 projected from, and the point +// of the requirement is to make that shape of claim impossible to emit. +func TestPayloadDecompositionBlock_RequiresTwoShapes(t *testing.T) { + tk := runnerTokenizer(t) + + d, err := DecomposePayload(tk, decompCorpus(), "only@1") + if err != nil { + t.Fatalf("DecomposePayload: %v", err) + } + if _, err := PayloadDecompositionBlockFor([]*PayloadDecomposition{d}); err == nil { + t.Error("a single-shape decomposition block must be an error — one corpus is what " + + "spec 102 projected from") + } +} diff --git a/bench/replay.go b/bench/replay.go index 9134568c7..31e7b49eb 100644 --- a/bench/replay.go +++ b/bench/replay.go @@ -126,15 +126,25 @@ type ReplayOptions struct { // replay input is raw recorded traffic and must never be committed. RecordingPath string - // Fleet is the MANDATORY fleet input: the tool definitions the menu cost - // is computed from. Either a frozen corpus (the committed corpora already - // serve this) or a snapshot of a live proxy's catalog. + // Fleet is the frozen half of the MANDATORY fleet input: the tool + // definitions the menu cost is computed from, as a committed corpus. + // Exactly one of Fleet and LiveFleet must be supplied. Fleet *Corpus // FleetID names the fleet shape every figure is quoted for (IC-004). // Empty falls back to the corpus version. FleetID string + // LiveFleet is the live half of the fleet input: a running proxy's own + // catalog, read over MCP tools/list (mcptools.go). It is an ALTERNATIVE to + // Fleet, never a supplement — see ResolveFleet for why supplying both is + // an error rather than a preference. + // + // It is a structural FleetSource for the same reason Arms are structural + // EncodingArms: bench cannot import the packages that build the concrete + // producers, and a test must be able to satisfy the seam with a literal. + LiveFleet FleetSource + // Bodies is the loader's privacy posture. The ZERO VALUE is bodies-off, // which is the safe default and the one this file's contract is written // for; bodies-on is an explicit opt-in that warns. @@ -172,6 +182,87 @@ func ReplayArmNames() []string { return names } +// ResolveFleet picks the ONE fleet a replay is priced over, from the two ways +// of supplying one: a frozen corpus, or a live proxy's own catalog read over +// MCP tools/list (mcptools.go). +// +// Two rules, and they are the same rule seen from both sides. +// +// NEITHER given is ErrReplayFleetRequired. A menu is a property of the tool +// definitions and the activity export carries no fleet snapshot, so a +// recording on its own computes nothing at all. Adding a live source does NOT +// soften this: it adds a second way to supply a fleet, not permission to omit +// one. +// +// BOTH given is ErrReplayFleetAmbiguous. This one is easy to get wrong in the +// accommodating direction — prefer the live proxy, say, and warn. But every +// figure in the report is quoted for exactly one fleet shape, named by a single +// fleet_id, and a reader has no way to tell which input that id came from. A +// silent preference therefore does not produce a slightly-less-precise report; +// it produces a report whose provenance line is false. Refuse instead. +// +// The live fetch happens HERE rather than at the call site so that a live +// fleet's failure lands on the same code path as every other fleet failure. A +// caller that fell back to a frozen corpus on a failed fetch would answer a +// different question than the one that was asked, using the same headline. +func ResolveFleet(frozen *Corpus, frozenID string, live FleetSource) (*Corpus, string, error) { + hasFrozen := frozen != nil && len(frozen.Tools) > 0 + switch { + case hasFrozen && live != nil: + return nil, "", fmt.Errorf("%w: a frozen corpus and a live proxy catalog were both supplied, and every figure is quoted for "+ + "exactly one fleet shape — silently preferring one would make the report's fleet_id a claim about definitions it was not priced over", + ErrReplayFleetAmbiguous) + + case live != nil: + fleet, err := live.Fleet() + if err != nil { + return nil, "", fmt.Errorf("replay: read the live fleet: %w", err) + } + if fleet == nil || len(fleet.Tools) == 0 { + return nil, "", fmt.Errorf("%w: the live fleet source returned no tools", ErrReplayFleetRequired) + } + // Guard HERE, not only inside FetchToolsList. The guard's job is to + // stop a stubbed live fleet from being priced, and a check that lives + // in one implementation of FleetSource only protects that one + // implementation — any other live source reaches the tokenizer + // unchecked. ResolveFleet is where every live fleet passes exactly + // once, so it is where the property actually holds. + if err := guardToolSchemas("live fleet "+live.FleetID(), fleet.Tools); err != nil { + return nil, "", err + } + id := live.FleetID() + if id == "" { + id = fleet.Version + } + return fleet, id, nil + + case hasFrozen: + // DELIBERATELY UNGUARDED, and not an oversight — a review flagged the + // asymmetry with the live path as a defect and it is not one. + // + // A schema-less frozen corpus is a SUPPORTED input. corpus_v1 (spec + // 065) is 45 tools with zero schemas by design, and it is the default + // -corpus value; CountToolWithSchema is documented to count a + // schema-less tool identically to CountTool precisely so the two mix + // in one report. Applying the live guard here would reject the + // project's own default corpus. + // + // The asymmetry tracks a real difference in what the two sources can + // silently be. A live surface can be misconfigured out from under the + // operator — direct_tool_response_mode:"deferred" turns every upstream + // schema into a placeholder and under-counts the direct menu by ~39% + // with nothing on screen to say so. A committed corpus is a reviewed + // artifact whose shape was chosen; if it carries no schemas, that is a + // property someone can see in the diff. + return frozen, frozenID, nil + + default: + return nil, "", fmt.Errorf("%w: a menu is a property of the tool definitions and the activity export carries no fleet snapshot, "+ + "so a recording on its own computes nothing — pass a frozen corpus or a live-proxy catalog "+ + "(this is an error, not a degraded run)", ErrReplayFleetRequired) + } +} + // FleetResolver builds the loader's replayability predicate from a fleet // input: does the (server, tool) a record names still exist in the fleet this // replay scores against? @@ -240,18 +331,20 @@ func splitToolIdentity(serverName, toolName string) (server, tool string) { // RunReplay loads a recording, prices every mode cell's menu against the // supplied fleet, and returns the report block. // -// The fleet check comes FIRST, before the recording is even opened: refusing +// The fleet is resolved FIRST, before the recording is even opened: refusing // early is what makes the refusal a rule about the invocation rather than an -// accident of whichever file happened to be readable. +// accident of whichever file happened to be readable. Resolution also covers +// the live source (ResolveFleet), so a live fleet's stub guard fires before any +// raw recorded traffic is read off disk. func RunReplay(tk *Tokenizer, opts ReplayOptions) (*ReplayBlock, error) { if tk == nil { return nil, errors.New("replay: a tokenizer is required") } - if opts.Fleet == nil || len(opts.Fleet.Tools) == 0 { - return nil, fmt.Errorf("%w: a menu is a property of the tool definitions and the activity export carries no fleet snapshot, "+ - "so a recording on its own computes nothing — pass a frozen corpus or a live-proxy catalog "+ - "(this is an error, not a degraded run)", ErrReplayFleetRequired) + fleet, fleetID, err := ResolveFleet(opts.Fleet, opts.FleetID, opts.LiveFleet) + if err != nil { + return nil, err } + opts.Fleet, opts.FleetID = fleet, fleetID if opts.RecordingPath == "" { return nil, errors.New("replay: a recording path is required (mcpproxy activity export --format json)") } @@ -616,6 +709,15 @@ func replayFleetShape(tk *Tokenizer, opts ReplayOptions) (FleetShape, error) { id = opts.Fleet.Version } shape := FleetShape{ID: id, ToolCount: len(opts.Fleet.Tools)} + // Count the partially-stubbed remainder so a fleet that PASSED the guard + // still shows how much of it could not be priced. The guard only refuses + // an all-stub population; a 3-of-45 regression flows straight through and + // would otherwise leave no trace in the report. + for _, tl := range opts.Fleet.Tools { + if len(tl.Schema) == 0 || isStubSchema(tl.Schema) { + shape.SchemalessTools++ + } + } arm := opts.Arms[baselineArmName] if arm == nil { diff --git a/bench/replay_test.go b/bench/replay_test.go index f9e414130..cb023197d 100644 --- a/bench/replay_test.go +++ b/bench/replay_test.go @@ -427,7 +427,7 @@ func TestRunReplay_MenuCostsFollowTheSurface(t *testing.T) { if menu[CellDirectDeferred] >= menu[CellDirectFull] { t.Errorf("deferred direct rendering must cost less than full: %d vs %d", menu[CellDirectDeferred], menu[CellDirectFull]) } - if block.FleetShape.ToolCount != len(runnerCorpus().Tools) { + if block.FleetShape.ToolCount != len(liveFleetCorpus().Tools) { t.Errorf("the fleet shape travels with every figure; want %d tools, got %d", len(runnerCorpus().Tools), block.FleetShape.ToolCount) } } @@ -594,3 +594,151 @@ func TestRunReplay_SurfacesLoaderLevelAccounting(t *testing.T) { t.Error("a withheld cost must carry its reason, not just a total — the total alone is not actionable") } } + +// --- Spec 103: the live fleet source --------------------------------------- +// +// A replay may take its fleet from a running proxy instead of a frozen corpus +// (bench/mcptools.go). These three tests hold the invocation rules AT THE +// RunReplay boundary, because that is where a caller meets them: the mapping +// and stub-guard arithmetic is covered in mcptools_test.go. + +// liveFleetCorpus is the shape a LIVE surface actually serves: real input +// schemas on every tool. runnerCorpus() is the frozen schema-less shape, which +// the stub guard refuses off a live source by design. +func liveFleetCorpus() *Corpus { + return &Corpus{ + Version: "live:127.0.0.1:18421/mcp/all", + Tools: []Tool{ + {ToolID: "fs:read_file", Server: "fs", Name: "read_file", + Description: "Read the contents of a file from disk", + Schema: json.RawMessage(`{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}`)}, + {ToolID: "git:git_log", Server: "git", Name: "git_log", + Description: "Show recent commit history of a repository", + Schema: json.RawMessage(`{"type":"object","properties":{"repo_path":{"type":"string"},"max_count":{"type":"integer","default":10}},"required":["repo_path"]}`)}, + {ToolID: "time:get_current_time", Server: "time", Name: "get_current_time", + Description: "Get current time in a specific timezone", + Schema: json.RawMessage(`{"type":"object","properties":{"timezone":{"type":"string"}}}`)}, + }, + } +} + +// A live fleet drives a complete replay, and the block is quoted for the live +// fleet's id — not the corpus version it never had. +func TestRunReplay_AcceptsALiveFleetSource(t *testing.T) { + tk := runnerTokenizer(t) + opts := replayOptions(t) + opts.Fleet, opts.FleetID = nil, "" + // A real live fleet carries real schemas — runnerCorpus() is the frozen + // schema-less shape, which the stub guard rightly refuses off a live + // surface. Use a schema-bearing fleet so this exercises the happy path + // rather than the guard. + opts.LiveFleet = &fakeFleetSource{id: "live:127.0.0.1:18421/mcp/all", corpus: liveFleetCorpus()} + + block, err := RunReplay(tk, opts) + if err != nil { + t.Fatalf("a live fleet must be a valid fleet input: %v", err) + } + if block.FleetShape.ID != "live:127.0.0.1:18421/mcp/all" { + t.Errorf("the block must be quoted for the live fleet id, got %q", block.FleetShape.ID) + } + if block.FleetShape.ToolCount != len(liveFleetCorpus().Tools) { + t.Errorf("live fleet tool count not carried through: %d", block.FleetShape.ToolCount) + } + for _, cell := range block.Cells { + if cell.MenuTokens <= 0 { + t.Errorf("cell %s priced no menu over the live fleet", cell.CellID) + } + } +} + +// Two fleets is an error at the RunReplay boundary too: a report carries ONE +// fleet_id, and preferring one input silently would make it false. +func TestRunReplay_CorpusAndLiveProxyTogetherIsAnError(t *testing.T) { + tk := runnerTokenizer(t) + opts := replayOptions(t) // already carries a frozen corpus + opts.LiveFleet = &fakeFleetSource{id: "live:x", corpus: runnerCorpus()} + + block, err := RunReplay(tk, opts) + if !errors.Is(err, ErrReplayFleetAmbiguous) { + t.Fatalf("want ErrReplayFleetAmbiguous, got %v", err) + } + if block != nil { + t.Fatalf("a refused replay must return no block, got %+v", block) + } +} + +// The live source ADDS a way to supply a fleet. It does not make one optional: +// a nil source is still no fleet. +func TestRunReplay_NilLiveFleetLeavesTheFleetMandatory(t *testing.T) { + tk := runnerTokenizer(t) + opts := replayOptions(t) + opts.Fleet, opts.FleetID, opts.LiveFleet = nil, "", nil + + if _, err := RunReplay(tk, opts); !errors.Is(err, ErrReplayFleetRequired) { + t.Fatalf("want ErrReplayFleetRequired, got %v", err) + } +} + +// A live fleet whose schemas are stubbed must abort the whole run. +// +// The previous version of this test injected ErrStubToolSchemas from a fake +// source, which proved only that RunReplay propagates an error it was handed — +// it passed with guardToolSchemas deleted entirely. This one feeds REAL stubbed +// tool definitions through the real guard, so deleting the guard fails it. +func TestRunReplay_StubSchemaFleetRefusesTheWholeRun(t *testing.T) { + tk := runnerTokenizer(t) + opts := replayOptions(t) + opts.Fleet, opts.FleetID = nil, "" + opts.LiveFleet = &fakeFleetSource{ + id: "live:deferred", + corpus: &Corpus{ + Version: "live:deferred", + Tools: []Tool{ + // Exactly what /mcp/all serves under + // direct_tool_response_mode:"deferred": the schema is folded + // into the description and replaced by a bare placeholder. + {ToolID: "git:git_log", Server: "git", Name: "git_log", + Description: "Shows the commit logs", + Schema: json.RawMessage(`{"type":"object"}`)}, + {ToolID: "fs:read_file", Server: "fs", Name: "read_file", + Description: "Read a file", + Schema: json.RawMessage(`{"type":"object"}`)}, + }, + }, + } + + if _, err := RunReplay(tk, opts); !errors.Is(err, ErrStubToolSchemas) { + t.Fatalf("a stub-schema live fleet must abort the replay, got %v", err) + } +} + +// A PARTIALLY stubbed fleet passes the guard — the guard only refuses an +// all-stub population — so the report must at least record how much of it could +// not be priced. +// +// Without this the difference between a clean 45-tool fleet and one where 20 +// tools lost their schemas is invisible in the output, while the second quietly +// shrinks the baseline and inflates the headline. +func TestRunReplay_CountsPartiallyStubbedFleet(t *testing.T) { + tk := runnerTokenizer(t) + partial := liveFleetCorpus() + // One of the three loses its schema: a partial regression, not a stubbed + // surface, so the run proceeds. + partial.Tools[1].Schema = json.RawMessage(`{"type":"object"}`) + + opts := replayOptions(t) + opts.Fleet, opts.FleetID = nil, "" + opts.LiveFleet = &fakeFleetSource{id: "live:partial", corpus: partial} + + block, err := RunReplay(tk, opts) + if err != nil { + t.Fatalf("a partially stubbed fleet still runs: %v", err) + } + if block.FleetShape.SchemalessTools != 1 { + t.Errorf("the report must record the unpriceable remainder; want 1, got %d", + block.FleetShape.SchemalessTools) + } + if block.FleetShape.ToolCount != 3 { + t.Errorf("tool count %d", block.FleetShape.ToolCount) + } +} diff --git a/bench/reportv2.go b/bench/reportv2.go index 05adaf4f5..410baa6d3 100644 --- a/bench/reportv2.go +++ b/bench/reportv2.go @@ -316,6 +316,16 @@ type FleetShape struct { ToolCount int `json:"tool_count"` MeanDefinitionTokens float64 `json:"mean_definition_tokens,omitempty"` P95DefinitionTokens int `json:"p95_definition_tokens,omitempty"` + // SchemalessTools counts tools in this fleet that carried no priceable + // input schema. + // + // A fleet is refused outright only when NO tool has a real schema. A + // PARTIAL regression — 3 of 45, or 20 of 45 — passes that guard and prices + // the stubbed ones at nothing, quietly shrinking the baseline and + // inflating the headline. Without this count the report carries no trace + // of it, so the reader cannot tell a clean fleet from a partly stubbed + // one, and a drift from 3 to 20 between runs is invisible. + SchemalessTools int `json:"schemaless_tools,omitempty"` } // ReplayExclusion counts the supplied sessions dropped for one reason (FR-003, @@ -500,8 +510,14 @@ type PayloadDecompositionRow struct { // The four attributions, as percentages of the definition payload. ShareNamesPct float64 `json:"share_names_pct"` ShareDescriptionsPct float64 `json:"share_descriptions_pct"` - ShareAnnotationsPct float64 `json:"share_annotations_pct"` - ShareSchemasPct float64 `json:"share_schemas_pct"` + // ShareAnnotationsPct is a POINTER because null and zero mean different + // things here. The frozen corpora carry no annotations field, so a + // corpus-based decomposition cannot measure this share at all — and a 0 + // would read as "measured, and annotations cost nothing", which is the + // silent-zero failure the rest of this report works to avoid. nil marshals + // to null: not measured. + ShareAnnotationsPct *float64 `json:"share_annotations_pct"` + ShareSchemasPct float64 `json:"share_schemas_pct"` // AchievableCeilingPct is recomputed for THIS shape. Carrying a // previously published ceiling forward is precisely the error spec 102 // made when it projected from a single corpus. diff --git a/bench/reportv2_test.go b/bench/reportv2_test.go index 8748b913a..894a717e5 100644 --- a/bench/reportv2_test.go +++ b/bench/reportv2_test.go @@ -382,6 +382,7 @@ func sampleAgentLoopBlock() *AgentLoopBlock { // samplePayloadDecomposition attributes definition cost across two fleet // shapes, each with its own recomputed ceiling and spec-102 verdict. func samplePayloadDecomposition() *PayloadDecompositionBlock { + annShare45, annShare527 := 1.5, 0.9 delta := -12.5 return &PayloadDecompositionBlock{ AccountingSource: AccountingSource{Kind: AccountingKindTokenizer, Identity: "cl100k_base"}, @@ -391,7 +392,7 @@ func samplePayloadDecomposition() *PayloadDecompositionBlock { Provenance: ProvenanceMeasured, ShareNamesPct: 3.1, ShareDescriptionsPct: 21.4, - ShareAnnotationsPct: 1.5, + ShareAnnotationsPct: &annShare45, ShareSchemasPct: 74.0, AchievableCeilingPct: 68.2, Spec102Verdict: Spec102Confirmed, @@ -401,7 +402,7 @@ func samplePayloadDecomposition() *PayloadDecompositionBlock { Provenance: ProvenanceMeasured, ShareNamesPct: 2.2, ShareDescriptionsPct: 30.8, - ShareAnnotationsPct: 0.9, + ShareAnnotationsPct: &annShare527, ShareSchemasPct: 66.1, AchievableCeilingPct: 55.7, Spec102Verdict: Spec102Corrected, diff --git a/specs/083-discovery-profiler/contracts/report-v2.schema.json b/specs/083-discovery-profiler/contracts/report-v2.schema.json index 4b79ea4d1..6ca11dc9e 100644 --- a/specs/083-discovery-profiler/contracts/report-v2.schema.json +++ b/specs/083-discovery-profiler/contracts/report-v2.schema.json @@ -886,7 +886,11 @@ "type": "number" }, "share_annotations_pct": { - "type": "number" + "type": [ + "number", + "null" + ], + "$comment": "NULL means the annotation share was not measurable from this corpus shape, which is different from zero. The frozen corpora carry no annotations field, while spec 102 measured the production wire payload that does. Emitting 0 would read as 'measured, and annotations cost nothing' — the silent-zero failure the exclusion accounting exists to prevent." }, "share_schemas_pct": { "type": "number" @@ -992,6 +996,11 @@ }, "p95_definition_tokens": { "type": "integer" + }, + "schemaless_tools": { + "type": "integer", + "minimum": 0, + "$comment": "Tools in this fleet carrying no priceable input schema. A fleet is refused only when NO tool has a real schema; a PARTIAL regression passes that guard, prices the stubbed tools at nothing and inflates the headline. This count is the only trace such a regression leaves." } } }, diff --git a/specs/103-token-bench/tasks.md b/specs/103-token-bench/tasks.md index 75daea902..94d825a37 100644 --- a/specs/103-token-bench/tasks.md +++ b/specs/103-token-bench/tasks.md @@ -264,16 +264,16 @@ recomputed per corpus and an explicit `confirmed`/`corrected` verdict. ### Tests for User Story 4 ⚠️ -- [ ] T050 [US4] In `bench/payloaddecomp_test.go`: the four shares sum to the whole payload -- [ ] T051 [US4] In `bench/payloaddecomp_test.go`: the achievable ceiling is recomputed per +- [x] T050 [US4] In `bench/payloaddecomp_test.go`: the four shares sum to the whole payload +- [x] T051 [US4] In `bench/payloaddecomp_test.go`: the achievable ceiling is recomputed per corpus and never carried forward as a constant — that carry-forward is precisely the error spec 102 made. Same file as the task above, so not parallel ### Implementation for User Story 4 -- [ ] T052 [US4] Implement the decomposition in `bench/payloaddecomp.go` over at least two fleet +- [x] T052 [US4] Implement the decomposition in `bench/payloaddecomp.go` over at least two fleet shapes (the 45-tool reference corpus and the 527-tool snapshot) -- [ ] T053 [US4] Emit the `payload_decomposition` block from `bench/payloaddecomp.go` with a +- [x] T053 [US4] Emit the `payload_decomposition` block from `bench/payloaddecomp.go` with a populated `accounting_source` and an explicit `spec102_verdict` (`confirmed` or `corrected`, with the delta)