diff --git a/ROADMAP.md b/ROADMAP.md index f674c642a..38ca4c8ff 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -50,6 +50,13 @@ graph LR tool_graph["Tool co-occurrence graph (experimental,…"] telemetry_identity["Telemetry identity & data quality (mach…"] telemetry_v7_churn["Telemetry v7: honest funnel + churn ins…"] + profiles_v3["Spec 108 profiles v3: scoped profiles,…"] + upstream_resilience["Spec 113 upstream resilience: OAuth ref…"] + pending_call_outcome["Spec 114 typed pending-call outcome + c…"] + mcp_credential_lifecycle["Spec 115 admin MCP credentials tool: cl…"] + config_commit_consistency["Spec 116 config commit consistency: one…"] + review_bound_approvals["Spec 117 review-bound approvals: durabl…"] + quarantine_approval_pilot["Quarantine/approval testing pilot: inde…"] scanner_v2 --> sandbox_isolation ux_audit --> action_log_transparency @@ -66,19 +73,26 @@ graph LR schema_deferred --> token_bench analytics_dashboard --> tool_graph telemetry_identity --> telemetry_v7_churn + upstream_resilience --> pending_call_outcome + profiles_v3 --> mcp_credential_lifecycle + config_commit_consistency --> review_bound_approvals + review_bound_approvals --> quarantine_approval_pilot classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff; classDef in_progress fill:#1f6feb,stroke:#0b3d91,color:#ffffff; + classDef in_review fill:#9a6700,stroke:#5c3d00,color:#ffffff; classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff; - class sandbox_isolation,scanner_v2,analytics_dashboard,scanner_simplification,schema_deferred,agent_scope_hardening done; - class ux_audit,action_log_transparency,token_bench,telemetry_identity,telemetry_v7_churn in_progress; + class sandbox_isolation,scanner_v2,analytics_dashboard,scanner_simplification,schema_deferred,agent_scope_hardening,upstream_resilience,mcp_credential_lifecycle,config_commit_consistency,review_bound_approvals,quarantine_approval_pilot done; + class ux_audit,action_log_transparency,token_bench,telemetry_identity,telemetry_v7_churn,profiles_v3 in_progress; + class pending_call_outcome in_review; class tpa_db,remote_access_tunnel,auto_routing_mode,tool_graph todo; ``` -**Independent epics** (15) — no cross-epic prerequisites; each stands alone: +**Independent epics** (16) — no cross-epic prerequisites; each stands alone: - 🔵 **Release qualification gate (auto-QA matrix blocks the tag)** — In progress · P0 - 🔵 **MCP protocol upgrade to 2026-07-28 revision** — In progress · P1 +- 🔵 **Spec 109 navigation, scope filters and cross-surface UX consistency** — In progress · P1 - 🔵 **Planning/docs truth automation** — In progress · P2 - 🔵 **Discovery-quality eval harness (Spec 065 second half)** — In progress · P3 - ⚪ **Windows native tray app** — Todo · P2 @@ -158,10 +172,8 @@ graph LR release_qa_gate_matrix --> release_qa_gate_consistency classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff; - classDef in_review fill:#9a6700,stroke:#5c3d00,color:#ffffff; classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff; - class release_qa_gate_matrix,release_qa_gate_playwright done; - class release_qa_gate_consistency in_review; + class release_qa_gate_matrix,release_qa_gate_playwright,release_qa_gate_consistency done; class release_qa_gate_macos todo; ``` @@ -170,7 +182,7 @@ graph LR | T1: tag-blocking release-gate workflow: server-type matrix (stdio/http/sse/docker/oauth) + invariants (activity-log/request-id, token+telemetry counters, quarantine flow, reconnect, upgrade-in-place), publish jobs gated on the verdict, scan-eval unconditional on tags | 🟢 Done | #819 | | T2: wire the Playwright Web UI sweep into the gate (currently manual-trigger only) | 🟢 Done | #1030 | | T3: macOS app smoke on a macos runner, advisory until 3 consecutive passes (today zero CI automation for the tray app) | ⚪ Todo | — | -| T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states) | 🟡 In review | — | +| T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states) | 🟢 Done | #1655 | @@ -317,6 +329,58 @@ Spec: [058-mcp-2026-upgrade](./specs/058-mcp-2026-upgrade/) +
+🔵 Spec 108 profiles v3: scoped profiles, client binding, scope-aware observability — In progress · P1 + +> Shipped 2026-09-26..10-05 as PRs 108-a..l. Left open: T126 live run of the six audit acceptance checks on one isolated instance across all surfaces (task below), and issue #1449 (macOS review follow-ups). + +Spec: [108-profiles-v3](./specs/108-profiles-v3/) + +```mermaid +graph LR + profiles_v3_impl["PRs 108-a..l: policy fields, enforcement, bin…"] + profiles_v3_live_acceptance["Live run of the six Spec 108 audit acceptance…"] + + + classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff; + classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff; + class profiles_v3_impl done; + class profiles_v3_live_acceptance todo; +``` + +| Task | Status | Refs | +| --- | --- | --- | +| PRs 108-a..l: policy fields, enforcement, bindings, CLI, MCP admin tool, Web UI, macOS, parity docs | 🟢 Done | #1380 #1411 #1430 #1436 #1441 #1442 #1443 #1444 #1448 #1456 #1500 | +| Live run of the six Spec 108 audit acceptance checks on one isolated instance (Web UI, macOS, CLI, MCP) — tasks.md T126 | ⚪ Todo | — | + +
+ +
+🔵 Spec 109 navigation, scope filters and cross-surface UX consistency — In progress · P1 + +> Shipped 2026-09-26..10-02 as PRs 109-a..m. Left open: T150 live run of every user story's independent test across surfaces (task below). + +Spec: [109-ux-navigation-consistency](./specs/109-ux-navigation-consistency/) + +```mermaid +graph LR + ux_navigation_impl["PRs 109-a..m: attention list, review queue, c…"] + ux_navigation_live_acceptance["Live run of every Spec 109 user story's indep…"] + + + classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff; + classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff; + class ux_navigation_impl done; + class ux_navigation_live_acceptance todo; +``` + +| Task | Status | Refs | +| --- | --- | --- | +| PRs 109-a..m: attention list, review queue, catalog-first Add, grouped navigation, parity docs | 🟢 Done | #1376 #1377 #1382 #1399 #1400 #1409 #1412 #1424 #1425 #1427 #1428 #1461 | +| Live run of every Spec 109 user story's independent test on one isolated instance — tasks.md T150 | ⚪ Todo | — | + +
+
🔵 Planning/docs truth automation — In progress · P2 @@ -372,6 +436,15 @@ graph LR
+
+🟡 Spec 114 typed pending-call outcome + call correlation for long-running stdio tool calls — In review · P2 + +> Spec-only PR #1536 open; implementation follows the spec review. Tracks issue #1321 (follow-up to #1317). + +PR: #1536 + +
+
⚪ tpa-db: versioned TPA signature database for the offline scanner — Todo · P1 @@ -607,6 +680,24 @@ graph LR
+
+🟢 Spec 116 config commit consistency: one commit section, atomic removal, upstream retirement — Done · P0 + +> Shipped 2026-10-12..10-14. Unticked tasks.md boxes are stale (the work merged); spec-gardener #1594 syncs them. + +Spec: [116-config-commit-consistency](./specs/116-config-commit-consistency/) · PR: #1590 #1600 #1608 #1612 #1621 + +
+ +
+🟢 Spec 117 review-bound approvals: durable baselines, hash-bound approvals, ordered tool inventories — Done · P0 + +> Shipped 2026-10-12..10-13 (PR-1..3). + +Spec: [117-review-bound-approvals](./specs/117-review-bound-approvals/) · PR: #1604 #1605 #1609 + +
+
🟢 Non-Docker sandbox isolation (Landlock) — Done · P1 · MCP-34 @@ -827,6 +918,33 @@ graph LR
+
+🟢 Spec 113 upstream resilience: OAuth refresh, discovery, call-error taxonomy, session re-init — Done · P1 + +> Shipped 2026-10-06..10-17 (113-a..f, plus #1650 serializing session re-init with in-flight requests). tasks.md checkboxes were never ticked (0/100); spec-gardener #1594 syncs them. + +Spec: [113-upstream-resilience](./specs/113-upstream-resilience/) · PR: #1495 #1498 #1499 #1501 #1502 #1504 #1510 #1650 + +
+ +
+🟢 Spec 115 admin MCP credentials tool: client credentials + agent token lifecycle — Done · P1 + +> Shipped 2026-10-10. #1579 (activity table fold + orphan seed honouring the Spec 115 guard) is an open follow-up fix. + +Spec: [115-mcp-credential-lifecycle](./specs/115-mcp-credential-lifecycle/) · PR: #1557 + +
+ +
+🟢 Quarantine/approval testing pilot: independent policy model, scenario inventory, mutation gate — Done · P1 + +> Shipped 2026-10-13..10-16: internal/qapilot model, S01-S26 scenarios, check ledger + mutation set run by quarantine-pilot.yml; #1634 fixed an approval-hash collision the pilot found. Docs: docs/development/quarantine-approval-pilot.md. + +PR: #1619 #1631 #1634 + +
+
🟢 Tray↔core decoupling: socket/REST API only, no config-file reads — Done · P2 @@ -892,8 +1010,11 @@ graph LR | Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — | | | | Telemetry v7: honest funnel + churn instrumentation | In progress | P1 | — | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) | | | MCP protocol upgrade to 2026-07-28 revision | In progress | P1 | 21/81 (26%) | [058-mcp-2026-upgrade](./specs/058-mcp-2026-upgrade/) | | +| Spec 108 profiles v3: scoped profiles, client binding, scope-aware observability | In progress | P1 | 194/195 (99%) | [108-profiles-v3](./specs/108-profiles-v3/) | | +| Spec 109 navigation, scope filters and cross-surface UX consistency | In progress | P1 | 257/258 (100%) | [109-ux-navigation-consistency](./specs/109-ux-navigation-consistency/) | | | Planning/docs truth automation | In progress | P2 | — | | | | Discovery-quality eval harness (Spec 065 second half) | In progress | P3 | — | [065-evaluation-foundation](./specs/065-evaluation-foundation/) | | +| Spec 114 typed pending-call outcome + call correlation for long-running stdio tool calls | In review | P2 | — | | #1536 | | tpa-db: versioned TPA signature database for the offline scanner | Todo | P1 | — | [101-tpa-db](./specs/101-tpa-db/) | | | Auto routing mode: budget-fitted tool surface per session (spec 104) | Todo | P1 | — | [104-auto-routing-mode](./specs/104-auto-routing-mode/) | | | Windows native tray app `MCP-43` | Todo | P2 | — | | | @@ -906,6 +1027,8 @@ graph LR | SDK v1 migration | Todo (parked) | P3 | — | | | | Upgrade awareness & guided update | Done | P0 | — | [079-upgrade-nudge](./specs/079-upgrade-nudge/) | | | Connect step trust: preview, visible backup, one-click undo | Done | P0 | — | [078-connect-trust-preview](./specs/078-connect-trust-preview/) | | +| Spec 116 config commit consistency: one commit section, atomic removal, upstream retirement | Done | P0 | 81/124 (65%) | [116-config-commit-consistency](./specs/116-config-commit-consistency/) | #1590 #1600 #1608 #1612 #1621 | +| Spec 117 review-bound approvals: durable baselines, hash-bound approvals, ordered tool inventories | Done | P0 | 46/50 (92%) | [117-review-bound-approvals](./specs/117-review-bound-approvals/) | #1604 #1605 #1609 | | Non-Docker sandbox isolation (Landlock) `MCP-34` | Done | P1 | — | | | | Spec 076 deterministic offline tool-scanner `MCP-3574` | Done | P1 | 22/24 (92%) | [076-deterministic-tool-scanner](./specs/076-deterministic-tool-scanner/) | | | Analytics dashboard as default page | Done | P1 | 25/26 (96%) | [069-observability-usage-graphs](./specs/069-observability-usage-graphs/) | | @@ -913,6 +1036,9 @@ graph LR | Scanner simplification (deterministic default, opt-in deep scan) | Done | P1 | 38/42 (90%) | [077-scanner-simplification](./specs/077-scanner-simplification/) | | | Deferred-schema serialization for the direct tools/list surface (spec 102) | Done | P1 | 89/89 (100%) | [102-schema-deferred](./specs/102-schema-deferred/) | #1063 | | Agent-token scope hardening: every MCP request authorized by its own scope (spec 105) | Done | P1 | 111/113 (98%) | [105-agent-scope-hardening](./specs/105-agent-scope-hardening/) | | +| Spec 113 upstream resilience: OAuth refresh, discovery, call-error taxonomy, session re-init | Done | P1 | 0/100 (0%) | [113-upstream-resilience](./specs/113-upstream-resilience/) | #1495 #1498 #1499 #1501 #1502 #1504 #1510 #1650 | +| Spec 115 admin MCP credentials tool: client credentials + agent token lifecycle | Done | P1 | 86/87 (99%) | [115-mcp-credential-lifecycle](./specs/115-mcp-credential-lifecycle/) | #1557 | +| Quarantine/approval testing pilot: independent policy model, scenario inventory, mutation gate | Done | P1 | — | | #1619 #1631 #1634 | | Tray↔core decoupling: socket/REST API only, no config-file reads | Done | P2 | — | | | | Spec 107 server edition SSO front door hardened for real IdPs | Done | P2 | 126/126 (100%) | [107-server-edition-sso-hardening](./specs/107-server-edition-sso-hardening/) | | diff --git a/roadmap.yaml b/roadmap.yaml index da1606b60..f6c137856 100644 --- a/roadmap.yaml +++ b/roadmap.yaml @@ -684,8 +684,9 @@ epics: depends_on: [release-qa-gate-matrix] - id: release-qa-gate-consistency title: "T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states)" - status: in_review - priority: P3 + status: done + pr: "#1655" + priority: P1 depends_on: [release-qa-gate-matrix] note: "BLOCKING gate entry invariant/surface-consistency (job surface-consistency): boots the five-type matrix, adds quarantined/disabled/unhealthy/live servers, and asserts CLI (`upstream list -o json`), MCP (`upstream_servers list`, an independent storage+manager projection) and the RENDERED Web UI (one long-lived Chromium page, e2e/web-ui-sweep/surface-snapshot.mjs) agree with core /api/v1/servers on enabled, quarantined, connection state, health.level/admin_state/status and tool_count. Three phases: settled, live (disable via REST, no page reload — tests the SSE/poll path), reload. Seeded negative: --seed-mismatch / workflow_dispatch seed_consistency_mismatch. Local: scripts/test-surface-consistency.sh. Tray recorded not-compared until T3." @@ -892,3 +893,87 @@ epics: depends_on: [] note: "The only thing standing between this epic and done. eval.yml sets continue-on-error for pull_request on the retrieval-D1 job because D1 depends on npx/uvx package fetches (a known flake source); the workflow's own comment plans the promotion after a green soak. Until then CI does not fail on a discovery regression on the PR path, which is exactly what FR-009 and SC-005 require." note: "IN PROGRESS — 2026-08-31 audit, corrected on cross-model review: both halves of the HARNESS shipped INDEPENDENTLY (not via token-bench-harness), but spec 065 is NOT fully met, so this is not done. FR-009 and SC-005 require CI to FAIL on a discovery regression beyond tolerance; the retrieval-D1 job is continue-on-error on pull requests, so on the PR path it does not fail — eval.yml itself records the promotion to PR-blocking as still open (MCP-742). A second, weaker tension to adjudicate rather than assume: CN-002 asks that scoring never run against a live drifting corpus, and D1 does boot a live mcpproxy serving 7 reference servers — but #931 pinned all seven upstreams to freeze-era versions and the job gates on the exact corpus ID set, so the corpus is reproducible in practice. Decide whether that satisfies CN-002 or whether a committed snapshot is required. Remaining work is therefore the gating promotion, not the harness. The earlier 'superseded / folded into token-bench-harness' framing was wrong on its own terms: token-bench-harness is still unbuilt, so nothing could have been folded into it. Security recall/FP half: cmd/scan-eval, backing the Spec 076/077 gate in eval.yml. Discovery-quality half: the eval.yml retrieval-d1 job boots mcpproxy and scores retrieval_golden_v1.json against a committed baseline at --tolerance 0.05 via the pinned external mcp-eval repo — note continue-on-error is scoped to github.event_name == 'pull_request', so the job is REPORT-ONLY on PRs (npx/uvx fetch flake) and BLOCKING on both the nightly schedule and manual workflow_dispatch runs. Promoting it to PR-blocking after a green soak is still open (MCP-742). NB the workflow's own inline comment says 'blocking on the nightly schedule' and omits workflow_dispatch. A second in-repo implementation lives in bench/: metrics.go defines RecallAtK/NDCGAtK, and the SC-003 recall@5 = 0.68 +/- 0.05 parity gate through the production Bleve index is asserted in bench/armindex_test.go (armindex.go supplies the production index wiring, not the assertion). Kept as a stable depends_on target; do not build a standalone harness." + + # ── Added 2026-10-17 truth-sync: specs shipped after the 2026-09-21 edit ── + - id: profiles-v3 + title: "Spec 108 profiles v3: scoped profiles, client binding, scope-aware observability" + status: in_progress + priority: P1 + spec: specs/108-profiles-v3 + depends_on: [] + note: "Shipped 2026-09-26..10-05 as PRs 108-a..l. Left open: T126 live run of the six audit acceptance checks on one isolated instance across all surfaces (task below), and issue #1449 (macOS review follow-ups)." + tasks: + - id: profiles-v3-impl + title: "PRs 108-a..l: policy fields, enforcement, bindings, CLI, MCP admin tool, Web UI, macOS, parity docs" + status: done + pr: ["#1380", "#1411", "#1430", "#1436", "#1441", "#1442", "#1443", "#1444", "#1448", "#1456", "#1500"] + depends_on: [] + - id: profiles-v3-live-acceptance + title: "Live run of the six Spec 108 audit acceptance checks on one isolated instance (Web UI, macOS, CLI, MCP) — tasks.md T126" + status: todo + priority: P2 + depends_on: [] + - id: ux-navigation-consistency + title: "Spec 109 navigation, scope filters and cross-surface UX consistency" + status: in_progress + priority: P1 + spec: specs/109-ux-navigation-consistency + depends_on: [] + note: "Shipped 2026-09-26..10-02 as PRs 109-a..m. Left open: T150 live run of every user story's independent test across surfaces (task below)." + tasks: + - id: ux-navigation-impl + title: "PRs 109-a..m: attention list, review queue, catalog-first Add, grouped navigation, parity docs" + status: done + pr: ["#1376", "#1377", "#1382", "#1399", "#1400", "#1409", "#1412", "#1424", "#1425", "#1427", "#1428", "#1461"] + depends_on: [] + - id: ux-navigation-live-acceptance + title: "Live run of every Spec 109 user story's independent test on one isolated instance — tasks.md T150" + status: todo + priority: P2 + depends_on: [] + - id: upstream-resilience + title: "Spec 113 upstream resilience: OAuth refresh, discovery, call-error taxonomy, session re-init" + status: done + priority: P1 + spec: specs/113-upstream-resilience + pr: ["#1495", "#1498", "#1499", "#1501", "#1502", "#1504", "#1510", "#1650"] + depends_on: [] + note: "Shipped 2026-10-06..10-17 (113-a..f, plus #1650 serializing session re-init with in-flight requests). tasks.md checkboxes were never ticked (0/100); spec-gardener #1594 syncs them." + - id: pending-call-outcome + title: "Spec 114 typed pending-call outcome + call correlation for long-running stdio tool calls" + status: in_review + priority: P2 + pr: "#1536" + depends_on: [upstream-resilience] + note: "Spec-only PR #1536 open; implementation follows the spec review. Tracks issue #1321 (follow-up to #1317)." + - id: mcp-credential-lifecycle + title: "Spec 115 admin MCP credentials tool: client credentials + agent token lifecycle" + status: done + priority: P1 + spec: specs/115-mcp-credential-lifecycle + pr: "#1557" + depends_on: [profiles-v3] + note: "Shipped 2026-10-10. #1579 (activity table fold + orphan seed honouring the Spec 115 guard) is an open follow-up fix." + - id: config-commit-consistency + title: "Spec 116 config commit consistency: one commit section, atomic removal, upstream retirement" + status: done + priority: P0 + spec: specs/116-config-commit-consistency + pr: ["#1590", "#1600", "#1608", "#1612", "#1621"] + depends_on: [] + note: "Shipped 2026-10-12..10-14. Unticked tasks.md boxes are stale (the work merged); spec-gardener #1594 syncs them." + - id: review-bound-approvals + title: "Spec 117 review-bound approvals: durable baselines, hash-bound approvals, ordered tool inventories" + status: done + priority: P0 + spec: specs/117-review-bound-approvals + pr: ["#1604", "#1605", "#1609"] + depends_on: [config-commit-consistency] + note: "Shipped 2026-10-12..10-13 (PR-1..3)." + - id: quarantine-approval-pilot + title: "Quarantine/approval testing pilot: independent policy model, scenario inventory, mutation gate" + status: done + priority: P1 + pr: ["#1619", "#1631", "#1634"] + depends_on: [review-bound-approvals] + note: "Shipped 2026-10-13..10-16: internal/qapilot model, S01-S26 scenarios, check ledger + mutation set run by quarantine-pilot.yml; #1634 fixed an approval-hash collision the pilot found. Docs: docs/development/quarantine-approval-pilot.md."