From 4b55a0026a66c53b2f7ab3cbece163608726c1be Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Sat, 29 Aug 2026 13:40:00 +0300 Subject: [PATCH 1/3] feat(settings): expose both serialization modes in the Web UI and tray MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tool_response_mode` (Spec 085) and `direct_tool_response_mode` (Spec 102) shipped with four ways to set them — the config file, an env var, a `serve` flag and the REST API — and zero UI surfaces. A tray-only operator could not reach `direct_tool_response_mode` at all, which is the entire point of the schema-deferred feature. Both now sit in Settings → General, directly under `routing_mode`, the knob they qualify. Neither carries a restart badge: the Go change detector appends them to ChangedFields without setting RequiresRestart, so they hot-apply — unlike `routing_mode` right above them, which genuinely needs a restart. Rendering the default honestly needed one new mechanism. Both keys are `omitempty` on the Go side, where an absent value means "full", so the config the API returns simply omits them until someone sets one — verified live: a fresh instance returns neither key. A native shows the real default instead of an empty box. Applied + // to both copies — otherwise the untouched field would read as dirty. + state.working = normalizeFieldDefaults(aliasServerEdition(clone(cfg))) + state.original = normalizeFieldDefaults(aliasServerEdition(clone(cfg))) configJson.value = JSON.stringify(cfg, null, 2) configStatus.value = { valid: true } loaded.value = true diff --git a/frontend/src/views/settings/fields.ts b/frontend/src/views/settings/fields.ts index 60fd9deda..d493a613c 100644 --- a/frontend/src/views/settings/fields.ts +++ b/frontend/src/views/settings/fields.ts @@ -50,6 +50,10 @@ export interface SettingField { valueKind?: ValueKind // extra format validation for text/secret fields optional?: boolean // when true, an empty value is valid (skips kind validation) resetDefault?: string // when set, render an inline "Reset to default" button that emits this value + // Value an absent/blank key actually resolves to on the Go side. Only needed + // for `omitempty` fields whose zero value is meaningful (the serialization + // modes: "" means "full"). See normalizeFieldDefaults below for why. + defaultValue?: string } export interface SettingsAccordion { @@ -252,6 +256,36 @@ export const GENERAL_FIELDS: SettingField[] = [ // serving the old surface. restart: true, }, + // Spec 085 / Spec 102 — the two SERIALIZATION axes. Neither is a routing + // mode: `routing_mode` picks the tool SURFACE, these two pick how each entry + // on that surface is rendered. Both hot-reload (the Go change detector sets + // requires_restart for neither), so no restart badge. Until this landed they + // were reachable only via the config file, an env var, a serve flag or the + // REST API — a tray-only user could not set them at all. + { + key: 'tool_response_mode', + docs: '/features/search-discovery#tool-response-mode', + label: 'Detail in tool-search results', + help: 'Applies to Retrieve mode. Full = every result carries its complete input schema. Compact = a one-line signature plus a first-sentence description, and the agent fetches a full schema on demand with describe_tool. Saves tokens; never changes which tools are found.', + control: 'select', + defaultValue: 'full', + options: [ + { value: 'full', label: 'Full — complete schemas (default)' }, + { value: 'compact', label: 'Compact — signatures, schema on demand' }, + ], + }, + { + key: 'direct_tool_response_mode', + docs: '/features/schema-deferred-direct-mode', + label: 'Detail in Direct-mode listings', + help: 'Applies to Direct mode, and to /mcp/all in any mode. Full = every tool is listed with its complete input schema. Deferred = name, description and a compact signature only, with schemas fetched on demand via describe_tool; a call whose arguments do not fit is rejected before it reaches the server, with the schema attached so the agent can correct itself.', + control: 'select', + defaultValue: 'full', + options: [ + { value: 'full', label: 'Full — complete schemas (default)' }, + { value: 'deferred', label: 'Deferred — signatures, schema on demand' }, + ], + }, { key: 'tools_limit', label: 'Search results limit', help: 'How many tools a single tool-search returns to the agent.', control: 'number', min: 1, max: 1000 }, { key: 'tool_response_limit', label: 'Max tool response size (characters)', help: 'Responses larger than this are truncated and cached so the agent can page through them. 0 = never truncate.', control: 'number', min: 0 }, { key: 'call_tool_timeout', label: 'Tool call timeout', help: 'How long to wait for a single tool call before giving up. e.g. 2m, 90s, 30s.', control: 'duration', placeholder: '2m' }, @@ -457,17 +491,42 @@ export const ADVANCED_ACCORDIONS: SettingsAccordion[] = [ * follows. */ export function restartRequiredLabels(): string[] { - const fields = [ + const labels: string[] = [] + for (const f of allCatalogFields()) { + if (f.restart && !labels.includes(f.label)) labels.push(f.label) + } + return labels +} + +/** Every field in the catalogue, across all sections and accordions. */ +export function allCatalogFields(): SettingField[] { + return [ ...GENERAL_FIELDS, ...SECURITY_FIELDS, ...SERVER_EDITION_FIELDS, ...ADVANCED_ACCORDIONS.flatMap((a) => a.fields), ] - const labels: string[] = [] - for (const f of fields) { - if (f.restart && !labels.includes(f.label)) labels.push(f.label) +} + +/** + * Fill in the resolved default for every catalogue field that declares a + * `defaultValue` and is currently blank in `cfg`. Mutates and returns cfg. + * + * The serialization-mode keys are `omitempty` on the Go side, where an absent + * value means "full" — so the config the API returns simply omits them until + * someone sets one. A native shows the real default instead of an empty box. Applied // to both copies — otherwise the untouched field would read as dirty. - state.working = normalizeFieldDefaults(aliasServerEdition(clone(cfg))) - state.original = normalizeFieldDefaults(aliasServerEdition(clone(cfg))) - configJson.value = JSON.stringify(cfg, null, 2) + const hydrated = hydrateConfigState(cfg) + state.working = hydrated.working + state.original = hydrated.original + formEpoch.value++ + // hydrated.raw, not cfg: the Raw tab must show the untouched response, + // and this makes that dependency explicit rather than relying on the + // hydration helper's internal cloning. + configJson.value = JSON.stringify(hydrated.raw, null, 2) configStatus.value = { valid: true } loaded.value = true maybePrefillInstructions() diff --git a/frontend/src/views/settings/fields.ts b/frontend/src/views/settings/fields.ts index d493a613c..9e1dd7807 100644 --- a/frontend/src/views/settings/fields.ts +++ b/frontend/src/views/settings/fields.ts @@ -519,6 +519,38 @@ export function allCatalogFields(): SettingField[] { * "full". Applied to BOTH the working copy and the last-saved snapshot so a * field nobody touched never counts as an unsaved change. */ +/** + * Back-compat for the teams -> server_edition rename (MCP-1086): if a config + * only carries the legacy `teams` key, mirror it onto `server_edition` so the + * form (which binds to `server_edition.*`) hydrates. Mutates and returns cfg. + */ +export function aliasServerEdition(cfg: any): any { + if (cfg && cfg.server_edition == null && cfg.teams != null) { + cfg.server_edition = cfg.teams + } + return cfg +} + +/** + * Build the Settings form state from a raw `GET /api/v1/config` response. + * + * Extracted from Settings.vue so the hydration invariant is testable directly + * rather than by matching source text. That invariant: + * + * - `working` and `original` BOTH get the resolved defaults, so a field nobody + * touched never counts as an unsaved change. + * - `raw` is the untouched response. The Raw JSON tab must show server truth, + * and both helpers above MUTATE their argument, so the clones matter. + */ +export function hydrateConfigState(cfg: any): { working: any; original: any; raw: any } { + const clone = (v: any) => (v == null ? v : JSON.parse(JSON.stringify(v))) + return { + working: normalizeFieldDefaults(aliasServerEdition(clone(cfg))), + original: normalizeFieldDefaults(aliasServerEdition(clone(cfg))), + raw: clone(cfg), + } +} + export function normalizeFieldDefaults(cfg: any): any { if (cfg == null || typeof cfg !== 'object') return cfg for (const f of allCatalogFields()) { diff --git a/frontend/tests/unit/settings-serialization-modes.spec.ts b/frontend/tests/unit/settings-serialization-modes.spec.ts index b287a60c2..bfdd57f1c 100644 --- a/frontend/tests/unit/settings-serialization-modes.spec.ts +++ b/frontend/tests/unit/settings-serialization-modes.spec.ts @@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { GENERAL_FIELDS, + hydrateConfigState, allCatalogFields, normalizeFieldDefaults, restartRequiredLabels, @@ -90,23 +91,14 @@ describe('normalizeFieldDefaults', () => { expect(cfg.direct_tool_response_mode).toBe('deferred') }) - // Settings.vue must normalize `working` AND `original` from the same - // response. Normalizing only the working copy is a live footgun: the value - // genuinely diverges from the raw response, so the untouched field would - // compare unequal and the section would open claiming an unsaved change. - // (Asserting the divergence is what gives the "normalize both" rule teeth — - // comparing two identically-normalized objects would pass unconditionally.) - it('diverges from the raw response, so both copies must be normalized', () => { + // The normalized copy genuinely diverges from the raw response — which is + // what makes the "normalize both snapshots" rule load-bearing. (Comparing two + // identically-normalized objects would pass unconditionally; that vacuous + // form is what cross-model review caught in the tray twin of this test.) + it('diverges from the raw response', () => { const response = { listen: '127.0.0.1:8080' } - const working = normalizeFieldDefaults({ ...response }) - - expect(getPath(working, 'tool_response_mode')).toBe('full') expect(getPath(response, 'tool_response_mode')).toBeUndefined() - - const original = normalizeFieldDefaults({ ...response }) - for (const f of allCatalogFields()) { - expect(getPath(working, f.key)).toEqual(getPath(original, f.key)) - } + expect(getPath(normalizeFieldDefaults({ ...response }), 'tool_response_mode')).toBe('full') }) it('leaves fields that declare no default untouched', () => { @@ -122,26 +114,65 @@ describe('normalizeFieldDefaults', () => { }) }) -// The catalogue is inert until Settings.vue applies it. There is no mount -// harness for that view, so pin the wiring at the source level — the same -// approach settings-deep-scan-field.spec.ts uses. Without this, dropping the -// normalize call would leave every test above green while the Settings page -// showed two empty dropdowns. -describe('Settings.vue wiring', () => { - const source = readFileSync( - resolve(__dirname, '../../src/views/Settings.vue'), - 'utf-8', - ) - - it('normalizes both the working copy and the last-saved snapshot', () => { - expect(source).toMatch(/state\.working = normalizeFieldDefaults\(/) - expect(source).toMatch(/state\.original = normalizeFieldDefaults\(/) - }) - - it('leaves the Raw JSON tab on the untouched server response', () => { - // configJson must serialize `cfg`, not a normalized clone — the Raw tab - // shows what the core actually holds, and normalization invents keys the - // config file does not contain. - expect(source).toMatch(/configJson\.value = JSON\.stringify\(cfg,/) +// hydrateConfigState is what Settings.vue's loadConfig() actually calls, so +// these assert the invariant itself rather than matching source text — a +// source-text match cannot tell whether `cfg` is still the untouched response +// by the time the Raw tab serializes it. +describe('hydrateConfigState', () => { + const response = () => ({ listen: '127.0.0.1:8080' }) + + it('normalizes both snapshots, so nothing reads as dirty on open', () => { + const { working, original } = hydrateConfigState(response()) + expect(getPath(working, 'tool_response_mode')).toBe('full') + expect(getPath(original, 'tool_response_mode')).toBe('full') + for (const f of allCatalogFields()) { + expect(getPath(working, f.key)).toEqual(getPath(original, f.key)) + } + }) + + it('keeps raw as the untouched server response for the Raw JSON tab', () => { + const { raw } = hydrateConfigState(response()) + expect(JSON.stringify(raw)).not.toContain('tool_response_mode') + expect(JSON.stringify(raw)).not.toContain('direct_tool_response_mode') + expect(getPath(raw, 'listen')).toBe('127.0.0.1:8080') + }) + + it('does not mutate its argument', () => { + const cfg = response() + hydrateConfigState(cfg) + expect(getPath(cfg, 'tool_response_mode')).toBeUndefined() + }) + + it('still aliases a legacy teams-keyed config onto server_edition', () => { + const { working } = hydrateConfigState({ teams: { enabled: true } }) + expect(getPath(working, 'server_edition.enabled')).toBe(true) + }) + + it('preserves an explicitly-set mode', () => { + const { working, raw } = hydrateConfigState({ direct_tool_response_mode: 'deferred' }) + expect(getPath(working, 'direct_tool_response_mode')).toBe('deferred') + expect(getPath(raw, 'direct_tool_response_mode')).toBe('deferred') + }) +}) + +// Regression: cross-model review found that a resolved default WAS writable. +// SettingsSection keeps a component-local `dirty` ref and dirtyKeys is the +// UNION of it and the working/original comparison. Reload replaces both +// snapshots but used to leave that ref populated, so: pick Compact -> Reload -> +// Save PATCHed `tool_response_mode: "full"` into a config that never had the +// key. Settings.vue now keys every section on a form epoch bumped per +// hydration, so the section remounts with an empty ref. +describe('Settings.vue rehydration (stale-dirty regression)', () => { + const source = readFileSync(resolve(__dirname, '../../src/views/Settings.vue'), 'utf-8') + + it('bumps a form epoch on every hydration', () => { + expect(source).toMatch(/formEpoch\.value\+\+/) + }) + + it('keys every SettingsSection on that epoch so the dirty ref cannot survive', () => { + const sections = source.match(/=70% target, with 38.9% the arithmetic ceiling even if both the schema and the signature were deleted. The projection assumed schemas dominate the payload; names, descriptions and annotations turn out to carry most of it. SC-001 is knowingly unmet and awaits a maintainer decision (restate per corpus shape, or re-target at a schema-heavy corpus) — see specs/102-schema-deferred/tasks.md T035a." - id: token-bench title: "Token-efficiency benchmark: measured savings, published results" diff --git a/specs/102-schema-deferred/plan.md b/specs/102-schema-deferred/plan.md index b4eb0220f..3cfe679c3 100644 --- a/specs/102-schema-deferred/plan.md +++ b/specs/102-schema-deferred/plan.md @@ -117,14 +117,14 @@ Technical approach, grounded in the code: ## Technical Context -**Language/Version**: Go 1.24 module toolchain (repo builds with local Go 1.25). Backend-only; no frontend, tray, or Swift changes. +**Language/Version**: Go 1.24 module toolchain (repo builds with local Go 1.25). Backend-only *as planned*; **superseded in delivery** — the follow-up settings work (#1082) added Vue (`frontend/src/views/settings/`) and Swift (`native/macos/.../Settings/`) changes to expose both serialization axes in the UI. The spec-102 implementation itself (#1063) was backend-only. **Primary Dependencies**: existing only — `github.com/mark3labs/mcp-go` v0.57.0 (tool registration, `SetTools` listChanged notification, `WithInstructions`), `github.com/santhosh-tekuri/jsonschema/v6` (already used by the Spec-085 validator being reused), `go.uber.org/zap`, stdlib. **No new dependencies.** **Storage**: none changed — BBolt and Bleve schemas untouched; the `directCatalog` is in-memory per-rebuild state; `ToolMetadata.OutputSchemaJSON` is already indexed (`internal/index/bleve.go:39`). **Testing**: `go test -race ./internal/...` (locally: `internal/server` with the CI skip regex); frozen-golden gates in `internal/server` (see Test strategy); E2E in `internal/server/e2e_test.go`; token measurements against the frozen 45-tool corpus via the spec-083/085 profiler pipeline. **Target Platform**: Linux/macOS/Windows core binary, personal + server editions — pure `internal/`, edition-agnostic (no `//go:build server` code touched; server-edition build verified). **Project Type**: Single Go project (core server). **Performance Goals**: Constitution I unaffected — serialization + registration-time work only; signatures come from the index-warmed cache via a pure read (`Peek`), no per-request compilation (FR-005); validation is memoized per tool hash and cheaper than the upstream round trip it prevents. -**Constraints**: FR-015 byte-stability of full-mode direct listings; FR-010 existing goldens pass unregenerated except the two enumerated FR-009 regens; FR-008 identical tool-set membership across modes; FR-013b fail-open validation; SC-001 ≥70% token reduction on the 45-tool corpus. +**Constraints**: FR-015 byte-stability of full-mode direct listings; FR-010 existing goldens pass unregenerated except the two enumerated FR-009 regens; FR-008 identical tool-set membership across modes; FR-013b fail-open validation; SC-001 ≥70% token reduction on the 45-tool corpus (**this constraint was NOT met — measured 29.7%, ceiling 38.9%; see spec.md SC-001**). **Scale/Scope**: ~6 packages touched: `internal/server` (core of the change), `internal/config`, `internal/runtime` (one clause), `internal/toolsig` (one method), `cmd/mcpproxy` (one flag), `bench/arms` (one measurement arm for the SC-001/SC-002 gates); plus docs and generated OpenAPI artifacts. ## Constitution Check @@ -135,7 +135,7 @@ Technical approach, grounded in the code: |-----------|------------| | **I. Performance at Scale** | PASS. BM25 search and indexing untouched. The direct rebuild already runs on upstream changes; deferred rendering *removes* per-entry schema unmarshal work and reads signatures via a pure cache lookup. Pre-dispatch validation is memoized per tool hash (existing validator). | | **II. Actor-Based Concurrency** | PASS (with one justified primitive). The catalog is written only on the single routing-refresh listener goroutine and read on request paths — an immutable snapshot behind an `atomic.Pointer` swap, replacing the existing mutex-guarded `directToolPermissions` map with the same read-mostly pattern Spec 085 justified for the signature cache. The catalog swap and mcp-go's `SetTools` remain two publications and cannot be made one transaction (mcp-go owns its registry read), so the resulting skew window is closed by ordering + deny-on-miss + registry-membership rules rather than papered over — D13/R13, with an interleaving test. See Complexity Tracking. | -| **III. Configuration-Driven Architecture** | PASS. One JSON config field with env override, serve flag, validation, and hot-reload (FR-001/FR-014); default preserves today's behavior; no tray state. | +| **III. Configuration-Driven Architecture** | PASS. One JSON config field with env override, serve flag, validation, and hot-reload (FR-001/FR-014); default preserves today's behavior; no tray state *in this spec*. (The follow-up #1082 added a read/write settings control in both the Web UI and the tray — still no tray-held state: the tray remains a REST client that reads and PATCHes core config.) | | **IV. Security by Default** | PASS. Serialization never changes membership (FR-008/FR-016): all discovery filters run before rendering. describe_tool on the direct surface applies listing-parity visibility *including the operation-permission tier the existing resolver lacks*, and never emits an existence-confirming reason code **or suggestion** for an id this session's listing omitted (FR-011). Parity is enforced on both sides (D10): the listing filters resolve through the catalog instead of re-parsing `__`, and both suggestion channels (`did_you_mean`, case-correction) are catalog-gated or suppressed — closing four concrete parity defects that exist in this tree today — three disclosure paths plus a wrong-safety-hint path (a listed destructive tool describing as `call_with: "read"`) — rather than opening any. Quarantine/approval gates on dispatch are untouched; validation is fail-open, never blocking a call a schemaless proxy would allow. The one place the design accepts staleness — an annotations-only change inside the two-publication window (Complexity Tracking, third row) — touches only listing/describe; call-time authorization is re-derived by the handler from its own captured annotations and is never one generation behind. | | **V. Test-Driven Development** | PASS. Every behavior lands test-first; the golden gates make surface drift a failing test by construction (see Test strategy). | | **VI. Documentation Hygiene** | PASS. `docs/configuration.md`, the feature doc under `docs/features/`, `CLAUDE.md` MCP-protocol line, and `make swagger` regen are in scope (see Documentation & wiring). | diff --git a/specs/102-schema-deferred/spec.md b/specs/102-schema-deferred/spec.md index 86b99a69b..1178263d7 100644 --- a/specs/102-schema-deferred/spec.md +++ b/specs/102-schema-deferred/spec.md @@ -2,7 +2,7 @@ **Feature Branch**: `102-schema-deferred` **Created**: 2026-08-24 -**Status**: Draft +**Status**: Draft — SHIPPED 2026-08-29 (all 89 tasks, PR #1063; settings UI in #1082). **SC-001 is measured and UNMET** (29.7% / 34.8% vs a >=70% target, 38.9% ceiling) and awaits a maintainer decision; see the SC-001 note under Measurable Outcomes. **Input**: User description: "schema_deferred routing (issue #971): direct-mode tools/list keeps ALL tool names + descriptions but defers inputSchema; agents fetch full schemas on demand via describe_tool. Built as a thin composition over the Spec 085 compact-router machinery (compact signatures, describe_tool, pre-dispatch validation + self-healing errors) per the maintainer's accepted direction (issue #971 comment, 2026-08-13)." **Related**: #971 (accepted-direction, 2026-08-13). Spec 085 (compact router — the reused machinery: signature cache, `describe_tool`, pre-dispatch validation, self-healing invalid-params errors). Specs 098/099 (preflight evaluator + `describe_tool` `check` mode — the full describe_tool contract travels wherever the tool is exposed). Industry precedent: Atlassian mcp-compressor "Low Compression" (94 tools: 17,600 → 3,900 tokens), Alibaba Qoder two-phase discovery, MCP client best-practices Catalog → Inspect → Execute. @@ -19,11 +19,11 @@ Per the maintainer's accepted direction, this is **not a new routing mode**: `ro ### User Story 1 - Deferred enumeration: see everything, pay for schemas only when needed (Priority: P1) -An agent connects to the direct surface of a proxy with deferred serialization enabled. `tools/list` returns every visible upstream tool — same `serverName__toolName` names, same annotations — but each entry carries the description plus its precompiled compact signature (`*` = required, `~` = lossy) instead of a full `inputSchema`; the declared input schema is a minimal permissive object. For a 100-tool fleet the listing drops from ~30K to roughly 3.5–5K tokens, and for flat (non-lossy) tools the agent can call directly from the signature with zero extra round trips. +An agent connects to the direct surface of a proxy with deferred serialization enabled. `tools/list` returns every visible upstream tool — same `serverName__toolName` names, same annotations — but each entry carries the description plus its precompiled compact signature (`*` = required, `~` = lossy) instead of a full `inputSchema`; the declared input schema is a minimal permissive object. For a 100-tool fleet the listing was projected to drop from ~30K to roughly 3.5–5K tokens; **the shipped measurement contradicts that projection** — a 527-tool snapshot went 99,918 → 65,138 tokens (~34.8%), because names, descriptions and annotations, not schemas, carry most of the payload (see the SC-001 note). For flat (non-lossy) tools the agent can still call directly from the signature with zero extra round trips, which held. **Why this priority**: This is the headline gap #971 names — full visibility at deferred-schema cost — and the entire token win of the feature. -**Independent Test**: Enable deferral on a fixture proxy with multiple upstream servers; fetch `tools/list` on the direct surface and assert: every tool present, no entry carries upstream schema properties, every entry's description ends with its compact signature, and total payload tokens are reduced ≥70% versus full mode on the frozen 45-tool reference corpus. +**Independent Test**: Enable deferral on a fixture proxy with multiple upstream servers; fetch `tools/list` on the direct surface and assert: every tool present, no entry carries upstream schema properties, every entry's description ends with its compact signature, and total payload tokens are reduced versus full mode on the frozen 45-tool reference corpus. (The ≥70% figure this test originally named was not met — measured 29.7%; the shipped gate in `bench/arms` asserts a 25% floor plus an upper guard, per T076.) **Acceptance Scenarios**: @@ -180,7 +180,22 @@ An operator flips deferred serialization on (or off) in config with the proxy ru ### Measurable Outcomes -- **SC-001**: On the frozen 45-tool reference corpus, deferred direct-surface `tools/list` payload is ≥70% smaller in tokens than full mode, measured by the spec-083 profiler pipeline; on a ~100-tool fleet the projection matches the #971 estimate (~30K → ~3.5–5K tokens, ≥85% reduction). +- **SC-001** (**NOT MET — measured, escalated, awaiting maintainer decision**): As written, this + required a ≥70% token reduction on the frozen 45-tool reference corpus and ≥85% on a ~100-tool + fleet. + + > **Measured outcome (2026-08-29, T035a).** 29.7% on the reference corpus and 34.8% on a 527-tool + > snapshot (99,918 → 65,138 tokens). The **arithmetic ceiling is 38.9%** — that is what deleting + > *both* the schema and the signature would yield — so no implementation of this design can reach + > 70% on this corpus shape. The target rested on Spec 083 profiling attributing ~77% of the payload + > to schemas; at the corpus shapes actually measured, names, descriptions and annotations dominate. + > + > This is recorded rather than quietly restated: the feature works and delivers a real ~30% saving, + > but the criterion as written is unreachable. **Two options for the maintainer:** restate the + > threshold per corpus shape, or re-target SC-001 at a schema-heavy corpus where the original + > premise holds. Until then the shipped gate (`bench/arms`, T076) asserts a 25% floor plus an upper + > guard that fails loudly if anything ever does clear 70%. See tasks.md T035a and the Phase 3 + > checkpoint for the full measurement table. - **SC-002**: ≥80% of corpus tools (the non-lossy share; Spec 085 lossy-rate gate <20%) are callable one-shot from the deferred listing — zero `describe_tool` round trips — verified against the corpus signature set. - **SC-003**: An agent whose guessed arguments fail validation completes the call in exactly one retry using only the error-embedded schema (no additional discovery calls), demonstrated in E2E. - **SC-004**: With deferral off, automated comparison shows direct-surface responses byte-identical to pre-feature behavior modulo the single enumerated surface delta (FR-010); the existing tool-surface goldens pass **unregenerated** (the delta lands on a surface they do not cover), and the new direct-surface built-in gate pins that delta. diff --git a/specs/102-schema-deferred/tasks.md b/specs/102-schema-deferred/tasks.md index 4ca9cf0d3..f39638609 100644 --- a/specs/102-schema-deferred/tasks.md +++ b/specs/102-schema-deferred/tasks.md @@ -280,7 +280,7 @@ So T004 gates Phase 1; T001–T003 gate their own dependents and are otherwise c ## Phase 7: Polish & cross-cutting -- [x] T073 Add the standalone direct built-in gate to `internal/server/toolslist_snapshot_test.go` with `testdata/toolslist_goldens/direct_mode_builtins.json`: zero upstream tools (listing = `describe_tool` only), **both** serialization modes, membership + byte-exact serialization, plus the direct server's instructions string captured with an empty `instructions` config so the bytes are deterministic. It MUST be standalone, not an entry in `toolsListGoldenSurfaces` — `_DeltaIsEnumerated` reads a frozen `pre099/.json` per listed surface, and direct mode has no pre-feature baseline. +- [x] T073 Add the standalone direct built-in gate to `internal/server/toolslist_snapshot_test.go` (**landed in a new file, `internal/server/toolslist_direct_builtins_test.go`, because the task itself requires the gate to be standalone rather than an entry in `toolsListGoldenSurfaces` — a separate file expresses that better than an orphan test in the shared snapshot file**) with `testdata/toolslist_goldens/direct_mode_builtins.json`: zero upstream tools (listing = `describe_tool` only), **both** serialization modes, membership + byte-exact serialization, plus the direct server's instructions string captured with an empty `instructions` config so the bytes are deterministic. It MUST be standalone, not an entry in `toolsListGoldenSurfaces` — `_DeltaIsEnumerated` reads a frozen `pre099/.json` per listed surface, and direct mode has no pre-feature baseline. - [x] T074 In the same gate, assert over `p.directServer.ListTools()` **and** a real `tools/list` driven through the direct server on a session with no agent token, treating any difference between the two sets as a failure. `ListTools()` is the registration map; `handleListTools` serves `filteredTools(ctx)`, and `directServer` is the one routing-mode server carrying `WithToolFilter`s — so registered and served can diverge on exactly this server. Sort by tool name before serializing: `ListTools()` returns a map and Go randomizes iteration order. - [x] T075 [P] Add the deferred-direct arm to `bench/arms/` plus its `bench/arms/testdata/*_golden.txt` render golden, following the registry contract in `specs/083-discovery-profiler/contracts/arm-interface.md` and `bench/arms/arm.go`. The existing arms all encode `retrieve_tools` result sets; none renders a direct `tools/list`. - [x] T076 Run the SC-001/SC-002 token gates over the frozen 45-tool corpus with the spec-083 profiler's pinned tokenizer: assert ≥70% payload reduction and ≥80% one-shot-callable (non-lossy) share, and record the measured numbers in the PR description. @@ -321,9 +321,12 @@ So T004 gates Phase 1; T001–T003 gate their own dependents and are otherwise c - [x] T083 Verify the five frozen gates land as predicted: gates 1 and 3 pass **unregenerated**; gate 2 shows exactly the two enumerated regens; gate 4 passes with the extended `describePlainDelta`; gate 5 is re-measured, not assumed. A diff anywhere else is a regression to fix, not a baseline to update. **Verified, not assumed.** All five pass. Golden churn across the entire branch is exactly: - the two enumerated regens (`default_server.json`, `retrieve_tools_mode.json`) plus four NEW - files (`direct_full_prefeature.golden.json`, the two `direct_mode_builtins_*.json`, and the - bench arm golden). `code_execution_mode.json` and both `pre099/` baselines are untouched — + the two enumerated regens (`default_server.json`, `retrieve_tools_mode.json`) plus three NEW + files (`direct_full_prefeature.golden.json`, `toolslist_goldens/direct_mode_builtins.json`, and + the bench arm golden `bench/arms/testdata/directdeferred_golden.txt`). *(Corrected 2026-08-29: + this note originally said "four NEW files … the two `direct_mode_builtins_*.json`". + `git show 9aef8d6f1 --diff-filter=A` lists three; there is a single builtins golden, which + covers both serialization modes in one file rather than one file per mode.)* `code_execution_mode.json` and both `pre099/` baselines are untouched — which is what confines the change to the two surfaces that register describe_tool. `go test -race ./internal/...` is green except `TestResolveDockerStatusResolvableAndWorking`,