Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ROADMAP.md
Original file line number Diff line number Diff line change
Expand Up @@ -772,7 +772,7 @@ graph LR
| Web UI + macOS app UX audit | In progress | P0 | — | | |
| Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | — | [081-release-qa-gate](./specs/081-release-qa-gate/) | |
| Action log / transparency — info at a glance | In progress | P1 | — | | |
| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 33/64 (52%) | [103-token-bench](./specs/103-token-bench/) | #1141 |
| Token-efficiency benchmark: measured savings, published results | In progress | P1 | 37/64 (58%) | [103-token-bench](./specs/103-token-bench/) | #1141 |
| Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — | | |
| Telemetry v7: honest funnel + churn instrumentation | In progress | P1 | — | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) | |
| Planning/docs truth automation | In progress | P2 | — | | |
Expand Down Expand Up @@ -914,4 +914,4 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—`
| [100-prompt-rugpull-baseline](./specs/100-prompt-rugpull-baseline/) | — | — |
| [101-tpa-db](./specs/101-tpa-db/) | — | — |
| [102-schema-deferred](./specs/102-schema-deferred/) | `shipped` | 89/89 (100%) |
| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 33/64 (52%) |
| [103-token-bench](./specs/103-token-bench/) | `in-flight` | 37/64 (58%) |
62 changes: 55 additions & 7 deletions bench/cmd/bench/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,19 @@
// go run ./bench/cmd/bench -replay /path/outside/the/repo/activity.jsonl \
// -corpus-v2 specs/083-discovery-profiler/datasets/corpus_v2.tools.json -out bench/results
//
// The fleet input is not optional. A menu is a property of the tool
// definitions and the activity export carries no fleet snapshot, so a
// recording on its own computes nothing — `-replay` without `-corpus-v2` or
// `-livemcptool` is an error, not a degraded run.
// The fleet may instead be read live from a running proxy's direct MCP
// surface, so an operator can price their OWN fleet without exporting a corpus
// first (Spec 103):
//
// go run ./bench/cmd/bench -replay /path/outside/the/repo/activity.jsonl \
// -proxy http://127.0.0.1:8080 -api-key KEY -out bench/results
//
// The fleet input is not optional, and it is not plural. A menu is a property
// of the tool definitions and the activity export carries no fleet snapshot,
// so a recording on its own computes nothing — `-replay` with no fleet flag at
// all is an error, not a degraded run. Supplying TWO (say -corpus-v2 and
// -proxy) is also an error: every figure is quoted for exactly one fleet
// shape, and silently preferring one would make the reported fleet_id false.
//
// A LAP lint artifact (`uvx --from lap-score==0.8.0 lap lint --json`) merges
// into either report via -lap-json. Reports land in bench/results/
Expand Down Expand Up @@ -91,10 +100,22 @@
// the EXISTING corpus flags (-corpus-v2 / -livemcptool) rather than a new
// one: a replay fleet is a tool corpus, and a second way to name one would
// be a second loader to keep honest.
replayPath := flag.String("replay", "", "path to an activity JSONL export (mcpproxy activity export --format json) to recompute cost over; REQUIRES a fleet input (-corpus-v2 or -livemcptool) — a recording alone computes nothing. The file must live OUTSIDE this repository and is never committed")
replayPath := flag.String("replay", "", "path to an activity JSONL export (mcpproxy activity export --format json) to recompute cost over; REQUIRES exactly one fleet input (-corpus-v2, -livemcptool, or -proxy to read a running proxy's own catalog over MCP tools/list) — a recording alone computes nothing, and two fleets is an error. The file must live OUTSIDE this repository and is never committed")
replayBodies := flag.Bool("replay-bodies-on-unmasked", false, "replay: read recorded request/response bodies. OFF by default. The activity export path does NOT mask, so this reads raw unmasked user traffic; bodies are tokenized inside the loader and only counts leave it")
flag.Parse()

// -proxy carries a non-empty DEFAULT, so its value cannot tell a replay
// run whether an operator actually asked for a live fleet. flag.Visit
// reports only what was set on the command line, which is the difference
// between "read this proxy's catalog" and "a default nobody typed" —
// without it, every -replay run would try to open a socket.
proxySet := false
flag.Visit(func(f *flag.Flag) {
if f.Name == "proxy" {
proxySet = true
}
})

// -live and -replay are different measurements over different inputs, and
// -live is checked first. Accepting both would silently run live mode and
// discard -replay with no output and no warning — a benchmark that quietly
Expand Down Expand Up @@ -132,6 +153,9 @@
outDir: *outDir,
corpusV2Path: *corpusV2Path,
livemcptoolPath: *livemcptoolPath,
liveProxy: *proxy,
liveProxySet: proxySet,
apiKey: *apiKey,
})
return
}
Expand Down Expand Up @@ -474,6 +498,14 @@
outDir string
corpusV2Path string
livemcptoolPath string

// liveProxy is the base URL a live fleet is read from, and liveProxySet
// records whether the operator actually typed -proxy. The two are separate
// because -proxy has a default: the URL alone cannot distinguish "read
// this proxy" from "nobody asked for a live fleet".
liveProxy string
liveProxySet bool
apiKey string
}

// runReplay is the Spec 103 US1 entry point: recompute per-cell menu cost and
Expand All @@ -491,6 +523,15 @@

fleet, fleetID := loadReplayFleet(opts.corpusV2Path, opts.livemcptoolPath)

// The live fleet source is CONSTRUCTED here but not read: RunReplay
// resolves it, so "exactly one fleet input" and "a live fetch failure is
// fatal" are decided in one place (bench.ResolveFleet) rather than being
// half-implemented by every caller.
var liveFleet bench.FleetSource
if opts.liveProxySet {
liveFleet = bench.NewLiveFleetSource(context.Background(), opts.liveProxy, opts.apiKey)
}

// Resolve exactly the arms the matrix needs to price a menu. A missing arm
// is fatal rather than a skipped row: an absent cell reads as "not
// applicable" when the truth would be "we could not measure it".
Expand All @@ -512,13 +553,14 @@
RecordingPath: opts.recording,
Fleet: fleet,
FleetID: fleetID,
LiveFleet: liveFleet,
Bodies: bodies,
Encoding: opts.encoding,
Arms: armsByName,
Warnf: func(format string, args ...any) { fmt.Fprintf(os.Stderr, "bench: "+format+"\n", args...) },
})
if err != nil {
log.Fatalf("bench: %v", err)

Check failure

Code scanning / CodeQL

Clear-text logging of sensitive information High

Sensitive data returned by an access to apiKey
flows to a logging call.
}

report := bench.ReplayReport(tk, block)
Expand All @@ -535,12 +577,18 @@
fmt.Fprintf(os.Stderr, "bench: the replay input is raw recorded traffic — delete %s when this run is finished\n", opts.recording)
}

// loadReplayFleet resolves the fleet input from the existing corpus flags.
// loadReplayFleet resolves the FROZEN fleet input from the corpus flags.
//
// Neither flag returns a nil fleet on purpose: the refusal belongs to
// bench.RunReplay, whose error explains WHY a recording alone computes
// nothing. Supplying both is refused here, because silently preferring one
// nothing. Two frozen corpora is refused here, because silently preferring one
// would make the reported fleet shape a coin toss.
//
// The LIVE fleet input (-proxy) is deliberately not resolved here. Frozen
// against live is the same ambiguity as frozen against frozen, but it is
// decided by bench.ResolveFleet so that the rule — and the decision to treat a
// failed live fetch as fatal rather than as grounds to fall back — is stated
// once, in the package a test can reach without a CLI.
func loadReplayFleet(corpusV2Path, livemcptoolPath string) (*bench.Corpus, string) {
if corpusV2Path != "" && livemcptoolPath != "" {
log.Fatalf("bench: replay takes ONE fleet input; -corpus-v2 and -livemcptool were both given, and every figure is quoted for exactly one fleet shape")
Expand Down
9 changes: 8 additions & 1 deletion bench/live_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,14 @@ func isStubSchema(raw json.RawMessage) bool {
Properties map[string]json.RawMessage `json:"properties"`
}
if err := json.Unmarshal(raw, &s); err != nil {
return false
// A schema that will not decode as an object carries no properties to
// price, so it is a stub for costing purposes. Returning false here —
// as an earlier version did — let a schema of `true`, `[]` or `0` pass
// as REAL and be priced at about one token, which is the
// semantically-empty-but-syntactically-different bypass around every
// guard downstream. Not reachable from mcpproxy today, which emits
// object schemas, but the guard should not depend on that.
return true
}
return len(s.Properties) == 0
}
Expand Down
64 changes: 63 additions & 1 deletion bench/mcpcaller.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"sync/atomic"

"github.com/mark3labs/mcp-go/client"
"github.com/mark3labs/mcp-go/client/transport"
Expand All @@ -16,8 +17,25 @@ import (
// through the SAME pipeline against the same live index. The serialized
// response text it returns is exactly what an agent would receive — that is
// what gets tokenized.
//
// Spec 103 gave it a second job — ToolsListPage, the raw tools/list fetch the
// live fleet source is built on — because the handshake is the same one and a
// second connect+initialize would be a second thing to keep correct. The two
// jobs share the session; only ToolsListPage needs the transport handle.
type MCPRetrieveCaller struct {
client *client.Client

// trans is the SAME transport the client wraps, kept so tools/list can be
// issued at the JSON-RPC level. That is not a shortcut: mcp-go's typed
// decode reshapes input schemas (it drops unknown top-level keywords and
// always re-emits "properties"/"required"), which would erase the exact
// bytes bench/mcptools.go's stub guard reads. See mcptools.go's header.
trans transport.Interface

// rawID numbers the requests issued outside the client. The ids are
// STRINGS in their own namespace so they can never collide with the
// client's int64 counter on the shared session.
rawID atomic.Int64
}

// NewMCPRetrieveCaller connects and initializes an MCP client against mcpURL
Expand All @@ -29,6 +47,12 @@ func NewMCPRetrieveCaller(ctx context.Context, mcpURL string) (*MCPRetrieveCalle
}
c := client.NewClient(httpTransport)
if err := c.Start(ctx); err != nil {
// Close on this path too, symmetric with the Initialize failure below.
// Benign with the current StreamableHTTP transport, whose Start is a
// no-op without continuous listening — but the symmetry is what stops
// a listener goroutine leaking the day that option is enabled, and
// noticing it then would be much harder than writing it now.
_ = c.Close()
return nil, fmt.Errorf("mcp start %q: %w", mcpURL, err)
}

Expand All @@ -39,14 +63,52 @@ func NewMCPRetrieveCaller(ctx context.Context, mcpURL string) (*MCPRetrieveCalle
_ = c.Close()
return nil, fmt.Errorf("mcp initialize %q: %w", mcpURL, err)
}
return &MCPRetrieveCaller{client: c}, nil
return &MCPRetrieveCaller{client: c, trans: httpTransport}, nil
}

// Close shuts the underlying MCP client down.
func (m *MCPRetrieveCaller) Close() error {
return m.client.Close()
}

// ToolsListPage implements ToolsListPageFunc (mcptools.go): one tools/list
// round trip on the live session, returning the RAW JSON-RPC result.
//
// It goes through the transport rather than client.ListTools deliberately.
// ListTools returns a decoded *mcp.ListToolsResult, and mcp-go's
// ToolArgumentsSchema keeps only {$defs, type, properties, required,
// additionalProperties} while its marshaller always re-emits "properties" and
// "required" — so re-serializing a decoded result rewrites the Spec 102
// deferred placeholder {"type":"object"} into
// {"properties":{},"required":[],"type":"object"} and makes it indistinguish-
// able from a genuinely parameter-less tool. The stub guard exists precisely
// to tell those apart, so the raw bytes are load-bearing, not an optimization.
//
// Pagination is NOT handled here: this is one page, and the cursor loop lives
// in FetchToolsList where it is testable without a proxy.
func (m *MCPRetrieveCaller) ToolsListPage(ctx context.Context, cursor string) (json.RawMessage, error) {
params := struct {
Cursor string `json:"cursor,omitempty"`
}{Cursor: cursor}

resp, err := m.trans.SendRequest(ctx, transport.JSONRPCRequest{
JSONRPC: mcp.JSONRPC_VERSION,
ID: mcp.NewRequestId(fmt.Sprintf("bench-tools-list-%d", m.rawID.Add(1))),
Method: MCPToolsListMethod,
Params: params,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", MCPToolsListMethod, err)
}
if resp.Error != nil {
return nil, fmt.Errorf("%s: server error %d: %s", MCPToolsListMethod, resp.Error.Code, resp.Error.Message)
}
if len(resp.Result) == 0 {
return nil, fmt.Errorf("%s: server returned an empty result", MCPToolsListMethod)
}
return resp.Result, nil
}

// RetrieveFunc adapts the caller to the RetrieveToolsFunc signature RunFlipGates
// consumes.
func (m *MCPRetrieveCaller) RetrieveFunc(ctx context.Context) RetrieveToolsFunc {
Expand Down
Loading
Loading