From 6d94a87b2cdfbe6061948d977100215e967bddf0 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Sat, 17 Oct 2026 16:55:40 +0300 Subject: [PATCH 1/3] feat(release-gate): blocking surface-state consistency check (Spec 081 T4) New gate entry invariant/surface-consistency (job surface-consistency) replaces the reserved T4 slot. It boots the five-type server matrix, adds quarantined, disabled, unhealthy and live servers, and asserts that the CLI (`upstream list -o json`), MCP `upstream_servers list` and the rendered Web UI agree with the core's /api/v1/servers on enabled, quarantined, connection state, health.level/admin_state/status and tool counts. Three phases: settled, live (disable via REST with the page left open), reload. A divergence that survives the retries fails the gate with a per-field diff. The macOS tray is recorded as not-compared until T3. ## Changes - cmd/release-gate: consistency comparator + `consistency` subcommand - e2e/web-ui-sweep/surface-snapshot.mjs: long-lived Web UI reader - internal/gatereport: blocking invariant/surface-consistency entry - release-qa-gate.yml: surface-consistency job, verdict dependency, dry-run-only seed_consistency_mismatch input - scripts/test-surface-consistency.sh: local runner - docs, specs/081 tasks.md, roadmap ## Testing - comparator tests: agreement, stale CLI level, MCP state, Web UI chip and tool count, missing/extra servers, capture error, seed - workflow audit: job blocking, compares Web UI, seed gated to dispatch - local run (stdio/http/sse/oauth): pass; seeded webui mismatch: fail --- .github/workflows/release-qa-gate.yml | 105 +++- ROADMAP.md | 10 +- cmd/release-gate/consistency.go | 422 +++++++++++++++ cmd/release-gate/consistency_cmd.go | 503 ++++++++++++++++++ cmd/release-gate/consistency_test.go | 220 ++++++++ cmd/release-gate/consistency_workflow_test.go | 44 ++ cmd/release-gate/main.go | 2 + docs/development/release-gate.md | 58 +- e2e/web-ui-sweep/surface-snapshot.mjs | 91 ++++ internal/gatereport/gatereport.go | 21 +- internal/gatereport/gatereport_test.go | 2 +- roadmap.yaml | 3 +- scripts/test-surface-consistency.sh | 103 ++++ specs/081-release-qa-gate/tasks.md | 13 + 14 files changed, 1572 insertions(+), 25 deletions(-) create mode 100644 cmd/release-gate/consistency.go create mode 100644 cmd/release-gate/consistency_cmd.go create mode 100644 cmd/release-gate/consistency_test.go create mode 100644 cmd/release-gate/consistency_workflow_test.go create mode 100644 e2e/web-ui-sweep/surface-snapshot.mjs create mode 100755 scripts/test-surface-consistency.sh create mode 100644 specs/081-release-qa-gate/tasks.md diff --git a/.github/workflows/release-qa-gate.yml b/.github/workflows/release-qa-gate.yml index 55e48f577..4b08d536d 100644 --- a/.github/workflows/release-qa-gate.yml +++ b/.github/workflows/release-qa-gate.yml @@ -21,10 +21,13 @@ name: Release QA Gate # entry): # advisory/web-ui-sweep — T2 Playwright sweep against the candidate # -# T3/T4 extension slots (reserved manifest entries, recorded as -# not-run/"not-implemented-yet" until their stage lands): +# T4 landed as a BLOCKING entry: +# invariant/surface-consistency — CLI / MCP / Web UI agree with the core's +# /api/v1/servers on every server's state +# +# T3 extension slot (reserved manifest entry, recorded as +# not-run/"not-implemented-yet" until its stage lands): # reserved/macos-smoke — T3 macOS tray smoke (advisory, FR-019/020) -# reserved/surface-consistency — T4 REST/CLI/Web-UI/tray state agreement # See docs/development/release-gate.md. on: @@ -40,6 +43,10 @@ on: description: "Ref to qualify (dry run — publishes nothing, FR-001a)" required: false type: string + seed_consistency_mismatch: + description: "Dry-run only: prove T4 fails — override surface:server:field=value (e.g. cli:gate-cons-live:health.level=degraded)" + required: false + type: string permissions: contents: read @@ -409,6 +416,96 @@ jobs: retention-days: 1 if-no-files-found: warn + # --------------------------------------------------------------------------- + # T4 (US5, FR-018): surface-state consistency. Boots the same five-type + # server matrix (its matrix fragments go to a scratch dir — the + # matrix-invariants job reports those cells), then `release-gate + # consistency` adds quarantined/disabled/unhealthy/live servers and asserts + # that the CLI (`upstream list -o json`), the MCP `upstream_servers list` and + # the RENDERED Web UI (one long-lived Chromium page) agree with the core's + # /api/v1/servers on enabled, quarantined, connection state, health.level, + # health.admin_state, health.status and tool counts — settled, after a live + # disable with no page reload, and after a reload. BLOCKING: a post-retry + # divergence fails the gate with a per-field diff. The macOS tray is T3. + # + # Negative proof: dispatch this workflow with seed_consistency_mismatch set; + # the knob is read only on workflow_dispatch, never on a tag's workflow_call. + # --------------------------------------------------------------------------- + surface-consistency: + name: Surface consistency (CLI / MCP / Web UI vs core) + needs: build-candidate + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.ref || github.sha }} + + - name: Set up Go # build-fixture-image.sh builds the static fixture binary + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Set up Node.js # the Web UI reader drives Chromium via @playwright/test + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Download candidate binaries + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: gate-candidate + path: dist-bin + + - name: Stage binaries + run: chmod +x dist-bin/* + + - name: Install the locked Playwright + Chromium + run: | + cd e2e/web-ui-sweep + npm ci + npx playwright install --with-deps chromium + + - name: Build fixture Docker image (docker cell) + run: scripts/gate/build-fixture-image.sh + + - name: Run surface consistency check + env: + # Read only on a manual dry run; empty on every tag (workflow_call). + SEED_MISMATCH: ${{ github.event_name == 'workflow_dispatch' && inputs.seed_consistency_mismatch || '' }} + run: | + set -uo pipefail + WORK="${RUNNER_TEMP}/consistency-work" + mkdir -p "$WORK" "${GATE_REPORT_DIR}" + ./dist-bin/release-gate matrix \ + --binary ./dist-bin/mcpproxy \ + --fixture ./dist-bin/mcpfixture \ + --oauth-server ./dist-bin/oauthserver \ + --report-dir "${WORK}/matrix-report" \ + --state-file "${WORK}/gate-state.json" \ + --work-dir "$WORK" + echo "matrix exit=$? (cells are reported by matrix-invariants)" + SEED_ARGS=() + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${SEED_MISMATCH}" ]; then + SEED_ARGS=(--seed-mismatch "${SEED_MISMATCH}") + fi + ./dist-bin/release-gate consistency \ + --state-file "${WORK}/gate-state.json" \ + --report-dir "${GATE_REPORT_DIR}" \ + --webui-script "${GITHUB_WORKSPACE}/e2e/web-ui-sweep/surface-snapshot.mjs" \ + "${SEED_ARGS[@]}" + + - name: Upload report fragment + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gate-fragment-surface-consistency + path: ${{ env.GATE_REPORT_DIR }}/ + retention-days: 1 + if-no-files-found: warn + # --------------------------------------------------------------------------- # T2 (US3, FR-016): Playwright Web UI sweep against the Web UI served by the # CANDIDATE binary (embedded frontend, not a dev server), with a live stdio @@ -488,7 +585,7 @@ jobs: # --------------------------------------------------------------------------- verdict: name: Gate verdict - needs: [build-candidate, suite-api-e2e, suite-race, suite-scan-eval, matrix-invariants, web-ui-sweep] + needs: [build-candidate, suite-api-e2e, suite-race, suite-scan-eval, matrix-invariants, surface-consistency, web-ui-sweep] if: always() runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/ROADMAP.md b/ROADMAP.md index 0175782be..f674c642a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -158,9 +158,11 @@ graph LR release_qa_gate_matrix --> release_qa_gate_consistency classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff; + classDef in_review fill:#9a6700,stroke:#5c3d00,color:#ffffff; classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff; class release_qa_gate_matrix,release_qa_gate_playwright done; - class release_qa_gate_macos,release_qa_gate_consistency todo; + class release_qa_gate_consistency in_review; + class release_qa_gate_macos todo; ``` | Task | Status | Refs | @@ -168,7 +170,7 @@ graph LR | T1: tag-blocking release-gate workflow: server-type matrix (stdio/http/sse/docker/oauth) + invariants (activity-log/request-id, token+telemetry counters, quarantine flow, reconnect, upgrade-in-place), publish jobs gated on the verdict, scan-eval unconditional on tags | 🟢 Done | #819 | | T2: wire the Playwright Web UI sweep into the gate (currently manual-trigger only) | 🟢 Done | #1030 | | T3: macOS app smoke on a macos runner, advisory until 3 consecutive passes (today zero CI automation for the tray app) | ⚪ Todo | — | -| T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states) | ⚪ Todo | — | +| T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states) | 🟡 In review | — | @@ -884,7 +886,7 @@ graph LR | Epic | Status | Priority | Progress | Spec | PR | | --- | --- | --- | --- | --- | --- | | Web UI + macOS app UX audit | In progress | P0 | — | | | -| Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | — | [081-release-qa-gate](./specs/081-release-qa-gate/) | | +| Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | 3/4 (75%) | [081-release-qa-gate](./specs/081-release-qa-gate/) | | | Action log / transparency — info at a glance | In progress | P1 | — | | | | Token-efficiency benchmark: measured savings, published results | In progress | P1 | 62/64 (97%) | [103-token-bench](./specs/103-token-bench/) | | | Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — | | | @@ -1008,7 +1010,7 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—` | [078-connect-trust-preview](./specs/078-connect-trust-preview/) | — | — | | [079-upgrade-nudge](./specs/079-upgrade-nudge/) | — | — | | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) | — | — | -| [081-release-qa-gate](./specs/081-release-qa-gate/) | — | — | +| [081-release-qa-gate](./specs/081-release-qa-gate/) | `in-flight` | 3/4 (75%) | | [082-work-sessions](./specs/082-work-sessions/) | — | — | | [083-discovery-profiler](./specs/083-discovery-profiler/) | `in-flight` | 4/41 (10%) | | [084-toon-output](./specs/084-toon-output/) | `in-flight` | 40/43 (93%) | diff --git a/cmd/release-gate/consistency.go b/cmd/release-gate/consistency.go new file mode 100644 index 000000000..619022444 --- /dev/null +++ b/cmd/release-gate/consistency.go @@ -0,0 +1,422 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/health" +) + +// Spec 081 T4 (US5, FR-018) — surface-state consistency. +// +// The core's own projection, GET /api/v1/servers, is the reference. Every +// other surface is read through ITS OWN path and reduced to a surfaceRow, and +// compareSurfaces reports each (surface, server, field) where the surface +// disagrees with the core: +// +// surface read path fields compared +// cli `mcpproxy upstream list -o json` enabled, quarantined, connected, connection_state, +// (daemon client mode → REST rows) health.level, health.admin_state, health.status, tool_count +// mcp /mcp `upstream_servers` operation=list enabled, quarantined, connection_state, +// (storage + upstream manager + runtime health.level, health.admin_state, health.status +// health — an independent projection; (no tool_count: the MCP row does not carry one) +// see internal/server/mcp.go handleListUpstreams) +// webui rendered server cards on /ui/servers enabled (⋯ menu Enable/Disable), quarantined (⋯ Review), +// (e2e/web-ui-sweep/surface-snapshot.mjs) tool_count, webui.chip_class, webui.chip_text +// tray — Spec 081 T3 (macOS runner) not compared here; recorded as not-compared +// +// The Web UI never prints health.level as text (Spec 109 FR-011), so its +// expected row is DERIVED from the core row with the same rules ServerCard.vue +// applies (expectedWebUIRow): chip colour from admin_state/level/sign-in, chip +// text from summary → status label. + +// surfaceRow is one server's state as one surface shows it: field → value. +type surfaceRow map[string]string + +// surfaceSnapshot is one capture of one surface. Err is set when the surface +// could not be read at all (that is itself a disagreement). +type surfaceSnapshot struct { + Surface string `json:"surface"` + Rows map[string]surfaceRow `json:"rows,omitempty"` + Err string `json:"error,omitempty"` +} + +// fieldMismatch names the surface, server, field and both values (FR-018). +type fieldMismatch struct { + Surface string `json:"surface"` + Server string `json:"server"` + Field string `json:"field"` + Core string `json:"core"` + Got string `json:"got"` +} + +const ( + surfaceCore = "core" + surfaceCLI = "cli" + surfaceMCP = "mcp" + surfaceWebUI = "webui" + surfaceTray = "tray" +) + +const ( + fieldPresence = "presence" + fieldCapture = "capture" + fieldEnabled = "enabled" + fieldQuarantined = "quarantined" + fieldConnected = "connected" + fieldConnection = "connection_state" + fieldLevel = "health.level" + fieldAdmin = "health.admin_state" + fieldStatus = "health.status" + fieldToolCount = "tool_count" + fieldChipClass = "webui.chip_class" + fieldChipText = "webui.chip_text" + + // Inputs to the Web UI derivation; never compared directly. + hiddenSummary = "_health.summary" + hiddenHealthAction = "_health.action" + hiddenDiagnosticCode = "_diagnostic.code" +) + +// surfaceFields lists what each surface is compared on. +var surfaceFields = map[string][]string{ + surfaceCLI: {fieldEnabled, fieldQuarantined, fieldConnected, fieldConnection, fieldLevel, fieldAdmin, fieldStatus, fieldToolCount}, + surfaceMCP: {fieldEnabled, fieldQuarantined, fieldConnection, fieldLevel, fieldAdmin, fieldStatus}, + surfaceWebUI: {fieldEnabled, fieldQuarantined, fieldToolCount, fieldChipClass, fieldChipText}, +} + +// rawServer is the union of the server-row shapes REST, CLI and MCP emit. +type rawServer struct { + Name string `json:"name"` + Enabled *bool `json:"enabled"` + Quarantined *bool `json:"quarantined"` + Connected *bool `json:"connected"` + Status string `json:"status"` + ToolCount *int `json:"tool_count"` + ConnectionStatus *struct { + State string `json:"state"` + } `json:"connection_status"` + Health *struct { + Level string `json:"level"` + AdminState string `json:"admin_state"` + Status string `json:"status"` + Summary string `json:"summary"` + Action string `json:"action"` + } `json:"health"` + Diagnostic *struct { + Code string `json:"code"` + } `json:"diagnostic"` +} + +// row reduces a raw server to a surfaceRow. connState is the surface's own +// connection-state field (REST/CLI `status`, MCP connection_status.state). +func (s rawServer) row(connState string, withToolCount, withConnected bool) surfaceRow { + r := surfaceRow{} + if s.Enabled != nil { + r[fieldEnabled] = strconv.FormatBool(*s.Enabled) + } + if s.Quarantined != nil { + r[fieldQuarantined] = strconv.FormatBool(*s.Quarantined) + } + if withConnected && s.Connected != nil { + r[fieldConnected] = strconv.FormatBool(*s.Connected) + } + r[fieldConnection] = normalizeConnState(connState) + if withToolCount && s.ToolCount != nil { + r[fieldToolCount] = strconv.Itoa(*s.ToolCount) + } + if s.Health != nil { + r[fieldLevel] = s.Health.Level + r[fieldAdmin] = s.Health.AdminState + r[fieldStatus] = s.Health.Status + r[hiddenSummary] = s.Health.Summary + r[hiddenHealthAction] = s.Health.Action + } + if s.Diagnostic != nil { + r[hiddenDiagnosticCode] = s.Diagnostic.Code + } + return r +} + +// normalizeConnState folds the casing difference between the StateView +// (REST) and the managed client's State.String() (MCP: "Ready", "Pending Auth"). +func normalizeConnState(s string) string { + return strings.ReplaceAll(strings.ToLower(strings.TrimSpace(s)), " ", "_") +} + +func rowsByName(servers []rawServer, conv func(rawServer) surfaceRow) (map[string]surfaceRow, error) { + out := make(map[string]surfaceRow, len(servers)) + for _, s := range servers { + if s.Name == "" { + return nil, fmt.Errorf("server row without a name") + } + if _, dup := out[s.Name]; dup { + return nil, fmt.Errorf("server %q listed twice", s.Name) + } + out[s.Name] = conv(s) + } + return out, nil +} + +func restLikeRow(s rawServer) surfaceRow { return s.row(s.Status, true, true) } + +// coreRowsFromREST parses the GET /api/v1/servers envelope (the reference). +func coreRowsFromREST(body []byte) (map[string]surfaceRow, error) { + var env struct { + Data struct { + Servers []rawServer `json:"servers"` + } `json:"data"` + } + if err := json.Unmarshal(body, &env); err != nil { + return nil, fmt.Errorf("parse /api/v1/servers: %w", err) + } + return rowsByName(env.Data.Servers, restLikeRow) +} + +// cliRowsFromJSON parses `mcpproxy upstream list -o json` stdout. +func cliRowsFromJSON(out []byte) (map[string]surfaceRow, error) { + var servers []rawServer + if err := json.Unmarshal(out, &servers); err != nil { + return nil, fmt.Errorf("parse `upstream list -o json` output: %w (output: %s)", err, truncateStr(string(out), 300)) + } + return rowsByName(servers, restLikeRow) +} + +// mcpRowsFromText parses the text content of upstream_servers operation=list. +func mcpRowsFromText(text string) (map[string]surfaceRow, error) { + var env struct { + Servers []rawServer `json:"servers"` + } + if err := json.Unmarshal([]byte(text), &env); err != nil { + return nil, fmt.Errorf("parse upstream_servers list: %w (text: %s)", err, truncateStr(text, 300)) + } + return rowsByName(env.Servers, func(s rawServer) surfaceRow { + state := "" + if s.ConnectionStatus != nil { + state = s.ConnectionStatus.State + } + return s.row(state, false, false) + }) +} + +// webuiCard is one rendered server card (surface-snapshot.mjs output). +type webuiCard struct { + Name string `json:"name"` + ChipClass string `json:"chip_class"` + ChipText string `json:"chip_text"` + ToolCount *int `json:"tool_count"` + Enabled *bool `json:"enabled"` + Quarantined bool `json:"quarantined"` +} + +// webuiRowsFromJSON parses one surface-snapshot.mjs response line. +func webuiRowsFromJSON(line []byte) (map[string]surfaceRow, error) { + var resp struct { + OK bool `json:"ok"` + Error string `json:"error"` + Rows []webuiCard `json:"rows"` + } + if err := json.Unmarshal(line, &resp); err != nil { + return nil, fmt.Errorf("parse Web UI snapshot: %w (line: %s)", err, truncateStr(string(line), 300)) + } + if !resp.OK { + return nil, fmt.Errorf("web UI snapshot failed: %s", resp.Error) + } + out := make(map[string]surfaceRow, len(resp.Rows)) + for _, c := range resp.Rows { + if c.Name == "" { + return nil, fmt.Errorf("web UI card without a server name") + } + if _, dup := out[c.Name]; dup { + return nil, fmt.Errorf("web UI renders server %q twice", c.Name) + } + r := surfaceRow{ + fieldChipClass: c.ChipClass, + fieldChipText: c.ChipText, + fieldQuarantined: strconv.FormatBool(c.Quarantined), + } + if c.Enabled != nil { + r[fieldEnabled] = strconv.FormatBool(*c.Enabled) + } + if c.ToolCount != nil { + r[fieldToolCount] = strconv.Itoa(*c.ToolCount) + } + out[c.Name] = r + } + return out, nil +} + +// oauthReauthCodes mirrors frontend/src/utils/health.ts OAuthReauthCodes. +var oauthReauthCodes = map[string]bool{ + "MCPX_OAUTH_REAUTH_REQUIRED": true, + "MCPX_OAUTH_REFRESH_EXPIRED": true, + "MCPX_OAUTH_REFRESH_403": true, +} + +// expectedWebUIRow derives what ServerCard.vue must render for a core row +// (statusBadgeClass / statusText; oauthSignInState from utils/health.ts). +func expectedWebUIRow(core surfaceRow) surfaceRow { + code := core[hiddenDiagnosticCode] + signIn := oauthReauthCodes[code] || core[hiddenHealthAction] == health.ActionLogin || + code == "MCPX_OAUTH_LOGIN_REQUIRED" + admin := core[fieldAdmin] + + var chip string + switch { + case admin == health.StateDisabled: + chip = "badge-neutral" + case signIn: + chip = "badge-warning" + case admin == health.StateQuarantined: + chip = "badge-secondary" + default: + switch core[fieldLevel] { + case health.LevelHealthy: + chip = "badge-success" + case health.LevelDegraded: + chip = "badge-warning" + case health.LevelUnhealthy: + chip = "badge-error" + default: + chip = "badge-ghost" + } + } + + var text string + switch { + case signIn && admin != health.StateDisabled: + text = health.StatusLabel(health.StatusSignInRequired) + case core[hiddenSummary] != "": + text = core[hiddenSummary] + case core[fieldStatus] != "": + text = health.StatusLabel(core[fieldStatus]) + case core[fieldConnected] == "true": + text = "Connected" + default: + text = "Disconnected" + } + + return surfaceRow{ + fieldEnabled: core[fieldEnabled], + fieldQuarantined: core[fieldQuarantined], + fieldToolCount: core[fieldToolCount], + fieldChipClass: chip, + fieldChipText: text, + } +} + +// connectionStateMeaningful: a disabled or quarantined server has no live +// connection both projections are obliged to agree on ("Not Started" in the +// manager vs "disabled" in the StateView is not a user-visible divergence; +// the admin state and health fields cover those servers). +func connectionStateMeaningful(core surfaceRow) bool { + return core[fieldEnabled] == "true" && core[fieldQuarantined] == "false" +} + +// compareSurfaces compares every surface snapshot against the core rows. +// The result is sorted (surface, server, field) so diffs are stable. +func compareSurfaces(core map[string]surfaceRow, snaps []surfaceSnapshot) []fieldMismatch { + var out []fieldMismatch + for _, snap := range snaps { + if snap.Err != "" { + out = append(out, fieldMismatch{Surface: snap.Surface, Server: "*", Field: fieldCapture, + Core: "readable", Got: snap.Err}) + continue + } + fields := surfaceFields[snap.Surface] + for name, coreRow := range core { + got, ok := snap.Rows[name] + if !ok { + out = append(out, fieldMismatch{Surface: snap.Surface, Server: name, Field: fieldPresence, + Core: "present", Got: "absent"}) + continue + } + want := coreRow + if snap.Surface == surfaceWebUI { + want = expectedWebUIRow(coreRow) + } + for _, f := range fields { + if f == fieldConnection && !connectionStateMeaningful(coreRow) { + continue + } + if want[f] != got[f] { + out = append(out, fieldMismatch{Surface: snap.Surface, Server: name, Field: f, + Core: want[f], Got: got[f]}) + } + } + } + for name := range snap.Rows { + if _, ok := core[name]; !ok { + out = append(out, fieldMismatch{Surface: snap.Surface, Server: name, Field: fieldPresence, + Core: "absent", Got: "present"}) + } + } + } + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a.Surface != b.Surface { + return a.Surface < b.Surface + } + if a.Server != b.Server { + return a.Server < b.Server + } + return a.Field < b.Field + }) + return out +} + +// formatMismatches renders a per-field diff, one line per mismatch. +func formatMismatches(mm []fieldMismatch) string { + sorted := append([]fieldMismatch(nil), mm...) + sort.SliceStable(sorted, func(i, j int) bool { + if sorted[i].Surface != sorted[j].Surface { + return sorted[i].Surface < sorted[j].Surface + } + return sorted[i].Server < sorted[j].Server + }) + lines := make([]string, 0, len(sorted)) + for _, m := range sorted { + lines = append(lines, fmt.Sprintf("%s %s %s: core=%s %s=%s", m.Surface, m.Server, m.Field, m.Core, m.Surface, m.Got)) + } + return strings.Join(lines, "\n") +} + +// seedMismatch overrides one captured field on every capture so the check's +// negative path can be proven end-to-end (`--seed-mismatch`, workflow_dispatch +// input `seed_consistency_mismatch`). Never set on a tag run. +type seedMismatch struct { + Surface, Server, Field, Value string +} + +// parseSeedMismatch parses "surface:server:field=value". +func parseSeedMismatch(s string) (*seedMismatch, error) { + parts := strings.SplitN(s, ":", 3) + if len(parts) != 3 { + return nil, fmt.Errorf("seed %q: want surface:server:field=value", s) + } + fv := strings.SplitN(parts[2], "=", 2) + if len(fv) != 2 || fv[0] == "" || parts[1] == "" { + return nil, fmt.Errorf("seed %q: want surface:server:field=value", s) + } + if _, ok := surfaceFields[parts[0]]; !ok { + return nil, fmt.Errorf("seed %q: unknown surface %q (cli, mcp, webui)", s, parts[0]) + } + return &seedMismatch{Surface: parts[0], Server: parts[1], Field: fv[0], Value: fv[1]}, nil +} + +func applySeed(seed *seedMismatch, snaps []surfaceSnapshot) { + if seed == nil { + return + } + for i := range snaps { + if snaps[i].Surface != seed.Surface || snaps[i].Rows == nil { + continue + } + if row, ok := snaps[i].Rows[seed.Server]; ok { + row[seed.Field] = seed.Value + } + } +} diff --git a/cmd/release-gate/consistency_cmd.go b/cmd/release-gate/consistency_cmd.go new file mode 100644 index 000000000..99b2a533d --- /dev/null +++ b/cmd/release-gate/consistency_cmd.go @@ -0,0 +1,503 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/gatereport" +) + +// Mixed-state servers the consistency check adds on top of the matrix fleet +// (FR-018 "fixtures in a known mixed state"). The matrix servers are all +// connected; these add the other admin states and an unhealthy server. +const ( + consServerLive = "gate-cons-live" // healthy, then disabled live (no page reload) + consServerQuarantined = "gate-cons-quarantined" // admin_state quarantined + consServerDisabled = "gate-cons-disabled" // admin_state disabled + consServerBroken = "gate-cons-broken" // enabled, endpoint refuses → unhealthy +) + +type consistencyOpts struct { + StateFile string + ReportDir string + CLIBinary string + WebUIScript string + NoWebUI bool + SetupMixed bool + SettleTimeout time.Duration + Seed *seedMismatch + KeepCore bool +} + +func cmdConsistency(ctx context.Context, args []string) (bool, error) { + fs := flag.NewFlagSet("consistency", flag.ExitOnError) + stateFile := fs.String("state-file", "", "state file written by `matrix --state-file` (required)") + reportDir := fs.String("report-dir", "", "directory for report fragments (required)") + cliBinary := fs.String("cli-binary", "", "mcpproxy binary used for the CLI surface (default: the matrix core binary)") + webuiScript := fs.String("webui-script", "", "path to e2e/web-ui-sweep/surface-snapshot.mjs (required unless --no-webui)") + noWebUI := fs.Bool("no-webui", false, "do not compare the Web UI (recorded as not-compared; local runs without Node only)") + setupMixed := fs.Bool("setup-mixed-state", true, "add quarantined/disabled/unhealthy/live servers before comparing") + settle := fs.Duration("settle-timeout", 60*time.Second, "per-phase budget for the surfaces to converge") + seed := fs.String("seed-mismatch", "", "test the negative path: override surface:server:field=value on every capture") + keepCore := fs.Bool("keep-core", false, "leave the matrix core running afterwards") + if err := fs.Parse(args); err != nil { + return false, err + } + if *stateFile == "" || *reportDir == "" { + return false, fmt.Errorf("--state-file and --report-dir are required") + } + if *webuiScript == "" && !*noWebUI { + return false, fmt.Errorf("--webui-script is required (or pass --no-webui to record the Web UI as not-compared)") + } + opts := consistencyOpts{ + StateFile: *stateFile, ReportDir: *reportDir, CLIBinary: absOrEmpty(*cliBinary), + WebUIScript: absOrEmpty(*webuiScript), NoWebUI: *noWebUI, SetupMixed: *setupMixed, + SettleTimeout: *settle, KeepCore: *keepCore, + } + if *seed != "" { + s, err := parseSeedMismatch(*seed) + if err != nil { + return false, err + } + opts.Seed = s + } + return runConsistency(ctx, opts) +} + +func runConsistency(ctx context.Context, opts consistencyOpts) (bool, error) { + st, err := readState(opts.StateFile) + if err != nil { + return false, err + } + if opts.CLIBinary == "" { + opts.CLIBinary = st.CoreBinary + } + frag := runTimed(gatereport.EntrySurfaceConsistency, func(frag *gatereport.Fragment) error { + return checkSurfaceConsistency(ctx, st, opts, frag) + }) + if !opts.KeepCore { + teardownState(ctx, st) + } + if err := gatereport.WriteFragment(opts.ReportDir, frag); err != nil { + return false, fmt.Errorf("write fragment: %w", err) + } + logf("consistency: %s -> %s %s", frag.Name, frag.Status, frag.Reason) + return frag.Status == gatereport.StatusPass, nil +} + +// checkSurfaceConsistency runs the three phases: +// +// settled fixtures in the mixed state, Web UI freshly loaded +// live one server disabled through REST while the page stays open — +// the Web UI must follow via its SSE/poll path, not a reload +// reload the same state after a page reload (tells "stale live view" +// apart from "renders the state wrong") +func checkSurfaceConsistency(ctx context.Context, st *gateState, opts consistencyOpts, frag *gatereport.Fragment) error { + client := newClient(st.BaseURL, st.APIKey) + if err := client.statusOK(ctx); err != nil { + return &infraError{fmt.Errorf("matrix core unreachable at %s: %w", st.BaseURL, err)} + } + details := map[string]any{ + "reference": "core: GET /api/v1/servers", + "compared": []string{surfaceCLI, surfaceMCP}, + "not_compared": map[string]string{ + surfaceTray: "macOS tray is Spec 081 T3 (macos-smoke, advisory) — not part of this Linux comparison", + }, + "fields": surfaceFields, + } + frag.Details = details + if opts.Seed != nil { + details["seeded_mismatch"] = *opts.Seed + } + + if opts.SetupMixed { + step, err := timedStep("setup-mixed-state", func() error { return setupMixedState(ctx, client, st) }) + frag.Steps = append(frag.Steps, step) + if err != nil { + return err + } + } + + mcpSess, err := newMCPSession(ctx, st.BaseURL, "") + if err != nil { + return &infraError{fmt.Errorf("MCP session: %w", err)} + } + defer mcpSess.close() + + var web *webuiReader + if opts.NoWebUI { + details["not_compared"].(map[string]string)[surfaceWebUI] = "--no-webui (local run without Node/Playwright)" + } else { + details["compared"] = []string{surfaceCLI, surfaceMCP, surfaceWebUI} + web, err = startWebUIReader(ctx, opts.WebUIScript, st.BaseURL, st.APIKey) + if err != nil { + return &infraError{fmt.Errorf("start Web UI reader: %w", err)} + } + defer web.close() + } + capt := &capturer{client: client, mcp: mcpSess, web: web, cliBinary: opts.CLIBinary, + configPath: st.ConfigPath, seed: opts.Seed} + + var failures []string + runPhase := func(name string, webCmd string) []fieldMismatch { + var mm []fieldMismatch + step, perr := timedStep(name, func() error { + var err error + mm, err = capt.converge(ctx, webCmd, opts.SettleTimeout) + if err != nil { + return err + } + if len(mm) > 0 { + return fmt.Errorf("%d field(s) disagree with core after %s:\n%s", len(mm), opts.SettleTimeout, formatMismatches(mm)) + } + return nil + }) + frag.Steps = append(frag.Steps, step) + details["mismatches_"+name] = mm + details["captured_"+name] = map[string]any{surfaceCore: capt.lastCore, "surfaces": capt.lastSnaps} + if perr != nil { + failures = append(failures, name+": "+perr.Error()) + } + return mm + } + + if web != nil { + if _, err := web.do(ctx, "load"); err != nil { + return &infraError{fmt.Errorf("load /ui/servers: %w", err)} + } + } + runPhase("settled", "snapshot") + + if opts.SetupMixed { + if err := client.postJSON(ctx, "/api/v1/servers/"+url.PathEscape(consServerLive)+"/disable", map[string]any{}, nil); err != nil { + failures = append(failures, "live: disable "+consServerLive+": "+err.Error()) + } else { + liveMM := runPhase("live", "snapshot") + reloadMM := runPhase("reload", "reload") + if web != nil && hasSurface(liveMM, surfaceWebUI) && !hasSurface(reloadMM, surfaceWebUI) { + failures = append(failures, "the Web UI only showed the new state after a page reload — its live update path is stale") + } + } + } + + if len(failures) > 0 { + return errors.New(strings.Join(failures, "\n")) + } + return nil +} + +func hasSurface(mm []fieldMismatch, surface string) bool { + for _, m := range mm { + if m.Surface == surface { + return true + } + } + return false +} + +func timedStep(name string, fn func() error) (gatereport.Step, error) { + start := time.Now() + err := fn() + step := gatereport.Step{Name: name, Status: gatereport.StatusPass, DurationMS: time.Since(start).Milliseconds()} + if err != nil { + step.Status = gatereport.StatusFail + step.Reason = err.Error() + } + return step, err +} + +// setupMixedState adds the four consistency servers and waits until each has +// reached the state it exists to show, so the comparison runs over a fleet +// covering every admin state plus an unhealthy server. +func setupMixedState(ctx context.Context, c *Client, st *gateState) error { + yes, no := true, false + isolateOff := &isolationRequest{EnabledOverride: &no} + port, err := freePort() // nothing listens here: connection refused + if err != nil { + return &infraError{err} + } + reqs := []addServerRequest{ + {Name: consServerLive, Command: st.FixtureBinary, Args: []string{"--transport", "stdio"}, Protocol: "stdio", + Enabled: &yes, Quarantined: &no, Isolation: isolateOff}, + {Name: consServerQuarantined, Command: st.FixtureBinary, Args: []string{"--transport", "stdio"}, Protocol: "stdio", + Enabled: &yes, Quarantined: &yes, Isolation: isolateOff}, + {Name: consServerDisabled, Command: st.FixtureBinary, Args: []string{"--transport", "stdio"}, Protocol: "stdio", + Enabled: &no, Quarantined: &no, Isolation: isolateOff}, + {Name: consServerBroken, URL: fmt.Sprintf("http://127.0.0.1:%d/mcp", port), Protocol: "http", + Enabled: &yes, Quarantined: &no}, + } + for _, r := range reqs { + if err := c.addServer(ctx, r); err != nil { + return fmt.Errorf("add %s: %w", r.Name, err) + } + } + // REST add may still land a server in quarantine (security default); + // the live/broken servers must be trusted for their state to be visible. + for _, name := range []string{consServerLive, consServerBroken} { + if s, err := c.server(ctx, name); err == nil && s != nil && s.Quarantined { + if err := c.unquarantineServer(ctx, name); err != nil { + return fmt.Errorf("unquarantine %s: %w", name, err) + } + } + } + want := map[string]func(*serverInfo) bool{ + consServerLive: func(s *serverInfo) bool { + return s.Connected && s.ToolCount > 0 && s.Health != nil && s.Health.Level == "healthy" + }, + consServerQuarantined: func(s *serverInfo) bool { return s.Health != nil && s.Health.AdminState == "quarantined" }, + consServerDisabled: func(s *serverInfo) bool { return s.Health != nil && s.Health.AdminState == "disabled" }, + consServerBroken: func(s *serverInfo) bool { + return !s.Connected && s.Health != nil && s.Health.AdminState == "enabled" && s.Health.Level != "healthy" + }, + } + deadline := time.Now().Add(90 * time.Second) + for { + servers, err := c.servers(ctx) + pending := []string{} + if err != nil { + pending = append(pending, "list: "+err.Error()) + } else { + for name, ok := range want { + found := false + for i := range servers { + if servers[i].Name == name { + found = true + if !ok(&servers[i]) { + pending = append(pending, fmt.Sprintf("%s (connected=%v tools=%d health=%+v)", name, servers[i].Connected, servers[i].ToolCount, servers[i].Health)) + } + } + } + if !found { + pending = append(pending, name+" (absent)") + } + } + } + if len(pending) == 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("mixed state not reached within 90s: %s", strings.Join(pending, "; ")) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + } +} + +// capturer reads every surface once per attempt. +type capturer struct { + client *Client + mcp *mcpSession + web *webuiReader + cliBinary string + configPath string + seed *seedMismatch + + // lastCore/lastSnaps are the final attempt's reads, kept for the report. + lastCore map[string]surfaceRow + lastSnaps []surfaceSnapshot +} + +func (c *capturer) core(ctx context.Context) (map[string]surfaceRow, error) { + var data json.RawMessage + if err := c.client.getJSON(ctx, "/api/v1/servers", &data); err != nil { + return nil, err + } + return coreRowsFromREST([]byte(`{"data":` + string(data) + `}`)) +} + +func (c *capturer) cli(ctx context.Context) surfaceSnapshot { + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + cmd := exec.CommandContext(cctx, c.cliBinary, "upstream", "list", "-o", "json", "--config", c.configPath) + // Never let the caller's environment point the CLI at another daemon. + cmd.Env = filteredEnv(os.Environ(), "MCPPROXY_TRAY_ENDPOINT", "MCPPROXY_API_KEY", "MCPPROXY_OUTPUT", "MCPPROXY_LISTEN") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + return surfaceSnapshot{Surface: surfaceCLI, Err: fmt.Sprintf("%v: %s", err, truncateStr(stderr.String()+stdout.String(), 400))} + } + rows, err := cliRowsFromJSON(stdout.Bytes()) + if err != nil { + return surfaceSnapshot{Surface: surfaceCLI, Err: err.Error()} + } + return surfaceSnapshot{Surface: surfaceCLI, Rows: rows} +} + +func (c *capturer) mcpList(ctx context.Context) surfaceSnapshot { + text, err := c.mcp.callTool(ctx, "upstream_servers", map[string]any{"operation": "list"}) + if err != nil { + return surfaceSnapshot{Surface: surfaceMCP, Err: err.Error()} + } + rows, err := mcpRowsFromText(text) + if err != nil { + return surfaceSnapshot{Surface: surfaceMCP, Err: err.Error()} + } + return surfaceSnapshot{Surface: surfaceMCP, Rows: rows} +} + +func (c *capturer) webui(ctx context.Context, cmd string) surfaceSnapshot { + line, err := c.web.do(ctx, cmd) + if err != nil { + return surfaceSnapshot{Surface: surfaceWebUI, Err: err.Error()} + } + rows, err := webuiRowsFromJSON(line) + if err != nil { + return surfaceSnapshot{Surface: surfaceWebUI, Err: err.Error()} + } + return surfaceSnapshot{Surface: surfaceWebUI, Rows: rows} +} + +// converge captures every surface until they agree with the core or the +// budget runs out. An attempt only counts when the core did not move while +// the surfaces were being read (core-before == core-after) — otherwise a +// legitimate transition would read as a disagreement. The first Web UI read +// uses webCmd ("snapshot" or "reload"); later attempts only re-read the page. +func (c *capturer) converge(ctx context.Context, webCmd string, budget time.Duration) ([]fieldMismatch, error) { + deadline := time.Now().Add(budget) + var last []fieldMismatch + stableAttempts := 0 + for attempt := 1; ; attempt++ { + before, err := c.core(ctx) + if err != nil { + return nil, &infraError{fmt.Errorf("read core: %w", err)} + } + snaps := []surfaceSnapshot{c.cli(ctx), c.mcpList(ctx)} + if c.web != nil { + snaps = append(snaps, c.webui(ctx, webCmd)) + webCmd = "snapshot" + } + after, err := c.core(ctx) + if err != nil { + return nil, &infraError{fmt.Errorf("read core: %w", err)} + } + if reflect.DeepEqual(before, after) { + stableAttempts++ + applySeed(c.seed, snaps) + c.lastCore, c.lastSnaps = after, snaps + last = compareSurfaces(after, snaps) + if len(last) == 0 { + logf("consistency: surfaces agree with core (attempt %d, %d servers)", attempt, len(after)) + return nil, nil + } + logf("consistency: attempt %d: %d mismatch(es)\n%s", attempt, len(last), formatMismatches(last)) + } else { + logf("consistency: attempt %d: core state moved during capture, retrying", attempt) + } + if time.Now().After(deadline) { + if stableAttempts == 0 { + return nil, fmt.Errorf("core state never held still for one capture within %s", budget) + } + return last, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(3 * time.Second): + } + } +} + +func filteredEnv(env []string, drop ...string) []string { + out := make([]string, 0, len(env)) + for _, kv := range env { + keep := true + for _, d := range drop { + if strings.HasPrefix(kv, d+"=") { + keep = false + break + } + } + if keep { + out = append(out, kv) + } + } + return out +} + +// webuiReader drives e2e/web-ui-sweep/surface-snapshot.mjs: one long-lived +// headless Chromium page, one JSON line per command on stdin/stdout. Keeping +// the page open is what lets the live phase test the Web UI's update path. +type webuiReader struct { + cmd *exec.Cmd + stdin io.WriteCloser + lines chan []byte + stderr *bytes.Buffer +} + +func startWebUIReader(ctx context.Context, script, baseURL, apiKey string) (*webuiReader, error) { + if _, err := os.Stat(script); err != nil { + return nil, err + } + cmd := exec.CommandContext(ctx, "node", script) + cmd.Dir = filepath.Dir(script) // resolves @playwright/test from the sweep's node_modules + cmd.Env = append(filteredEnv(os.Environ(), "MCPPROXY_API_KEY", "MCPPROXY_BASE_URL"), + "MCPPROXY_BASE_URL="+baseURL, "MCPPROXY_API_KEY="+apiKey) + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr := &bytes.Buffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, err + } + r := &webuiReader{cmd: cmd, stdin: stdin, lines: make(chan []byte, 4), stderr: stderr} + go func() { + sc := bufio.NewScanner(stdout) + sc.Buffer(make([]byte, 1<<20), 16<<20) + for sc.Scan() { + r.lines <- append([]byte(nil), sc.Bytes()...) + } + close(r.lines) + }() + return r, nil +} + +func (r *webuiReader) do(ctx context.Context, command string) ([]byte, error) { + if _, err := io.WriteString(r.stdin, command+"\n"); err != nil { + return nil, fmt.Errorf("send %q: %w (stderr: %s)", command, err, truncateStr(r.stderr.String(), 400)) + } + select { + case line, ok := <-r.lines: + if !ok { + return nil, fmt.Errorf("web UI reader exited (stderr: %s)", truncateStr(r.stderr.String(), 600)) + } + return line, nil + case <-time.After(90 * time.Second): + return nil, fmt.Errorf("web UI reader: %q timed out", command) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (r *webuiReader) close() { + _, _ = io.WriteString(r.stdin, "quit\n") + _ = r.stdin.Close() + done := make(chan struct{}) + go func() { _ = r.cmd.Wait(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + _ = r.cmd.Process.Kill() + } +} diff --git a/cmd/release-gate/consistency_test.go b/cmd/release-gate/consistency_test.go new file mode 100644 index 000000000..06e3f89b0 --- /dev/null +++ b/cmd/release-gate/consistency_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "strings" + "testing" +) + +// restFixture is a /api/v1/servers payload in a known mixed state: one healthy +// server, one quarantined, one disabled, one unhealthy. +const restFixture = `{"success":true,"data":{"servers":[ + {"name":"stdio-ok","enabled":true,"quarantined":false,"connected":true,"status":"Ready","tool_count":3, + "health":{"level":"healthy","admin_state":"enabled","status":"ready","summary":"Connected (3 tools)","actions":[]}}, + {"name":"quar","enabled":true,"quarantined":true,"connected":false,"status":"Disconnected","tool_count":0, + "health":{"level":"healthy","admin_state":"quarantined","status":"needs_review","summary":"Quarantined for review","action":"approve","actions":["approve"]}}, + {"name":"off","enabled":false,"quarantined":false,"connected":false,"status":"disabled","tool_count":0, + "health":{"level":"healthy","admin_state":"disabled","status":"disabled","summary":"Disabled","action":"enable","actions":["enable"]}}, + {"name":"broken","enabled":true,"quarantined":false,"connected":false,"status":"Error","tool_count":0, + "health":{"level":"unhealthy","admin_state":"enabled","status":"error","summary":"Connection refused","action":"restart","actions":["restart"]}} +]}}` + +// The CLI's `upstream list -o json` prints the raw REST rows (sorted). +const cliFixture = `[ + {"name":"broken","enabled":true,"quarantined":false,"connected":false,"status":"Error","tool_count":0, + "health":{"level":"unhealthy","admin_state":"enabled","status":"error","summary":"Connection refused"}}, + {"name":"off","enabled":false,"quarantined":false,"connected":false,"status":"disabled","tool_count":0, + "health":{"level":"healthy","admin_state":"disabled","status":"disabled","summary":"Disabled"}}, + {"name":"quar","enabled":true,"quarantined":true,"connected":false,"status":"Disconnected","tool_count":0, + "health":{"level":"healthy","admin_state":"quarantined","status":"needs_review","summary":"Quarantined for review"}}, + {"name":"stdio-ok","enabled":true,"quarantined":false,"connected":true,"status":"Ready","tool_count":3, + "health":{"level":"healthy","admin_state":"enabled","status":"ready","summary":"Connected (3 tools)"}} +]` + +// MCP `upstream_servers list` text result (an independent read path). +const mcpFixture = `{"total":4,"servers":[ + {"name":"stdio-ok","enabled":true,"quarantined":false,"connection_status":{"state":"Ready"}, + "health":{"level":"healthy","admin_state":"enabled","status":"ready"}}, + {"name":"quar","enabled":true,"quarantined":true,"connection_status":{"state":"Disconnected"}, + "health":{"level":"healthy","admin_state":"quarantined","status":"needs_review"}}, + {"name":"off","enabled":false,"quarantined":false,"connection_status":{"state":"Not Started"}, + "health":{"level":"healthy","admin_state":"disabled","status":"disabled"}}, + {"name":"broken","enabled":true,"quarantined":false,"connection_status":{"state":"Error"}, + "health":{"level":"unhealthy","admin_state":"enabled","status":"error"}} +]}` + +// What the Web UI reader (e2e/web-ui-sweep/surface-snapshot.mjs) extracts +// from the rendered server cards. +const webuiFixture = `{"ok":true,"rows":[ + {"name":"stdio-ok","chip_class":"badge-success","chip_text":"Connected (3 tools)","tool_count":3,"enabled":true,"quarantined":false}, + {"name":"quar","chip_class":"badge-secondary","chip_text":"Quarantined for review","tool_count":0,"enabled":true,"quarantined":true}, + {"name":"off","chip_class":"badge-neutral","chip_text":"Disabled","tool_count":0,"enabled":false,"quarantined":false}, + {"name":"broken","chip_class":"badge-error","chip_text":"Connection refused","tool_count":0,"enabled":true,"quarantined":false} +]}` + +func mustSnapshots(t *testing.T, cli, mcp, webui string) (map[string]surfaceRow, []surfaceSnapshot) { + t.Helper() + core, err := coreRowsFromREST([]byte(restFixture)) + if err != nil { + t.Fatalf("parse REST: %v", err) + } + cliRows, err := cliRowsFromJSON([]byte(cli)) + if err != nil { + t.Fatalf("parse CLI: %v", err) + } + mcpRows, err := mcpRowsFromText(mcp) + if err != nil { + t.Fatalf("parse MCP: %v", err) + } + webRows, err := webuiRowsFromJSON([]byte(webui)) + if err != nil { + t.Fatalf("parse Web UI: %v", err) + } + return core, []surfaceSnapshot{ + {Surface: surfaceCLI, Rows: cliRows}, + {Surface: surfaceMCP, Rows: mcpRows}, + {Surface: surfaceWebUI, Rows: webRows}, + } +} + +func TestCompareSurfaces_AgreeingMixedState(t *testing.T) { + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + if len(core) != 4 { + t.Fatalf("core rows = %d, want 4", len(core)) + } + if mm := compareSurfaces(core, snaps); len(mm) != 0 { + t.Fatalf("agreeing surfaces reported mismatches: %+v", mm) + } +} + +func TestCompareSurfaces_CLIStaleHealthLevel(t *testing.T) { + stale := strings.Replace(cliFixture, `"level":"unhealthy"`, `"level":"healthy"`, 1) + core, snaps := mustSnapshots(t, stale, mcpFixture, webuiFixture) + mm := compareSurfaces(core, snaps) + want := fieldMismatch{Surface: surfaceCLI, Server: "broken", Field: fieldLevel, Core: "unhealthy", Got: "healthy"} + if len(mm) != 1 || mm[0] != want { + t.Fatalf("mismatches = %+v, want exactly %+v", mm, want) + } +} + +func TestCompareSurfaces_MCPConnectionStateDiverges(t *testing.T) { + bad := strings.Replace(mcpFixture, `"state":"Ready"`, `"state":"Connecting"`, 1) + core, snaps := mustSnapshots(t, cliFixture, bad, webuiFixture) + mm := compareSurfaces(core, snaps) + if len(mm) != 1 || mm[0].Surface != surfaceMCP || mm[0].Server != "stdio-ok" || mm[0].Field != fieldConnection { + t.Fatalf("mismatches = %+v, want one mcp/stdio-ok/%s", mm, fieldConnection) + } +} + +func TestCompareSurfaces_WebUIShowsQuarantinedAsHealthy(t *testing.T) { + bad := strings.Replace(webuiFixture, `"chip_class":"badge-secondary"`, `"chip_class":"badge-success"`, 1) + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, bad) + mm := compareSurfaces(core, snaps) + want := fieldMismatch{Surface: surfaceWebUI, Server: "quar", Field: fieldChipClass, Core: "badge-secondary", Got: "badge-success"} + if len(mm) != 1 || mm[0] != want { + t.Fatalf("mismatches = %+v, want exactly %+v", mm, want) + } +} + +func TestCompareSurfaces_WebUIStaleToolCount(t *testing.T) { + bad := strings.Replace(webuiFixture, `"tool_count":3`, `"tool_count":0`, 1) + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, bad) + mm := compareSurfaces(core, snaps) + if len(mm) != 1 || mm[0].Field != fieldToolCount || mm[0].Core != "3" || mm[0].Got != "0" { + t.Fatalf("mismatches = %+v, want one tool_count 3 vs 0", mm) + } +} + +func TestCompareSurfaces_MissingAndExtraServers(t *testing.T) { + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + delete(snaps[0].Rows, "off") // CLI lost a server + snaps[2].Rows["ghost"] = surfaceRow{fieldEnabled: "true"} // Web UI shows a removed one + mm := compareSurfaces(core, snaps) + var sawMissing, sawExtra bool + for _, m := range mm { + if m.Surface == surfaceCLI && m.Server == "off" && m.Field == fieldPresence && m.Got == "absent" { + sawMissing = true + } + if m.Surface == surfaceWebUI && m.Server == "ghost" && m.Field == fieldPresence && m.Core == "absent" { + sawExtra = true + } + } + if !sawMissing || !sawExtra || len(mm) != 2 { + t.Fatalf("mismatches = %+v, want missing cli/off and extra webui/ghost", mm) + } +} + +func TestCompareSurfaces_SurfaceCaptureErrorFails(t *testing.T) { + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + snaps[0] = surfaceSnapshot{Surface: surfaceCLI, Err: "exit status 1: no daemon"} + mm := compareSurfaces(core, snaps) + if len(mm) != 1 || mm[0].Surface != surfaceCLI || mm[0].Field != fieldCapture { + t.Fatalf("mismatches = %+v, want one cli capture failure", mm) + } +} + +func TestExpectedWebUIChip(t *testing.T) { + cases := []struct { + name string + row surfaceRow + want string + }{ + {"disabled", surfaceRow{fieldAdmin: "disabled", fieldLevel: "unhealthy"}, "badge-neutral"}, + {"quarantined", surfaceRow{fieldAdmin: "quarantined", fieldLevel: "healthy"}, "badge-secondary"}, + {"quarantined needs login", surfaceRow{fieldAdmin: "quarantined", hiddenHealthAction: "login"}, "badge-warning"}, + {"healthy", surfaceRow{fieldAdmin: "enabled", fieldLevel: "healthy"}, "badge-success"}, + {"degraded", surfaceRow{fieldAdmin: "enabled", fieldLevel: "degraded"}, "badge-warning"}, + {"unhealthy", surfaceRow{fieldAdmin: "enabled", fieldLevel: "unhealthy"}, "badge-error"}, + {"reauth diagnostic", surfaceRow{fieldAdmin: "enabled", fieldLevel: "unhealthy", hiddenDiagnosticCode: "MCPX_OAUTH_REFRESH_EXPIRED"}, "badge-warning"}, + {"unknown level", surfaceRow{fieldAdmin: "enabled", fieldLevel: "weird"}, "badge-ghost"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := expectedWebUIRow(tc.row)[fieldChipClass]; got != tc.want { + t.Fatalf("chip class = %q, want %q", got, tc.want) + } + }) + } +} + +func TestExpectedWebUIChipText(t *testing.T) { + // summary wins; sign-in overrides it unless disabled; else the status label. + if got := expectedWebUIRow(surfaceRow{fieldAdmin: "enabled", hiddenSummary: "Connected (3 tools)", fieldStatus: "ready"})[fieldChipText]; got != "Connected (3 tools)" { + t.Fatalf("summary text = %q", got) + } + if got := expectedWebUIRow(surfaceRow{fieldAdmin: "enabled", hiddenHealthAction: "login", hiddenSummary: "x"})[fieldChipText]; got != "Sign-in required" { + t.Fatalf("sign-in text = %q", got) + } + if got := expectedWebUIRow(surfaceRow{fieldAdmin: "enabled", fieldStatus: "ready"})[fieldChipText]; got == "" || got == "ready" { + t.Fatalf("status-label fallback = %q, want the human label", got) + } +} + +func TestApplySeedMismatch(t *testing.T) { + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + seed, err := parseSeedMismatch("cli:stdio-ok:health.level=degraded") + if err != nil { + t.Fatalf("parse seed: %v", err) + } + applySeed(seed, snaps) + mm := compareSurfaces(core, snaps) + want := fieldMismatch{Surface: surfaceCLI, Server: "stdio-ok", Field: fieldLevel, Core: "healthy", Got: "degraded"} + if len(mm) != 1 || mm[0] != want { + t.Fatalf("seeded mismatch = %+v, want %+v", mm, want) + } + for _, bad := range []string{"", "cli", "cli:x", "cli:x:field", "nope:x:f=v"} { + if _, err := parseSeedMismatch(bad); err == nil { + t.Fatalf("parseSeedMismatch(%q) accepted", bad) + } + } +} + +func TestMismatchReportIsSortedAndNamesBothValues(t *testing.T) { + mm := []fieldMismatch{ + {Surface: surfaceWebUI, Server: "b", Field: fieldLevel, Core: "x", Got: "y"}, + {Surface: surfaceCLI, Server: "a", Field: fieldLevel, Core: "healthy", Got: "degraded"}, + } + s := formatMismatches(mm) + if !strings.HasPrefix(s, "cli a health.level: core=healthy cli=degraded") { + t.Fatalf("formatted diff = %q", s) + } +} diff --git a/cmd/release-gate/consistency_workflow_test.go b/cmd/release-gate/consistency_workflow_test.go new file mode 100644 index 000000000..7b62340fa --- /dev/null +++ b/cmd/release-gate/consistency_workflow_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "path/filepath" + "strings" + "testing" +) + +// TestSurfaceConsistencyJobIsBlocking audits the T4 wiring in +// release-qa-gate.yml: the surface-consistency job exists, is not advisory, +// actually compares the Web UI, feeds the verdict job, and its seed knob can +// only be set on a manual dry run (never on the workflow_call tag path). +func TestSurfaceConsistencyJobIsBlocking(t *testing.T) { + wf := parseWorkflow(t, filepath.Join("..", "..", ".github", "workflows", "release-qa-gate.yml")) + j, ok := wf.Jobs["surface-consistency"] + if !ok { + t.Fatal("release-qa-gate.yml has no surface-consistency job") + } + if v := strings.TrimSpace(j.ContinueOnError.Value); v != "" && v != "false" { + t.Errorf("surface-consistency must block: continue-on-error=%q", v) + } + var run string + for _, s := range j.Steps { + if strings.Contains(s.Run, "release-gate consistency") { + run = s.Run + } + } + if run == "" { + t.Fatal("surface-consistency job never runs `release-gate consistency`") + } + if !strings.Contains(run, "--webui-script") || strings.Contains(run, "--no-webui") { + t.Errorf("surface-consistency must compare the Web UI (--webui-script, never --no-webui):\n%s", run) + } + if !strings.Contains(run, "workflow_dispatch") { + t.Errorf("the --seed-mismatch knob must be guarded to workflow_dispatch runs:\n%s", run) + } + verdict, ok := wf.Jobs["verdict"] + if !ok { + t.Fatal("no verdict job") + } + if !containsStr(verdict.Needs, "surface-consistency") { + t.Errorf("verdict job needs %v, missing surface-consistency", verdict.Needs) + } +} diff --git a/cmd/release-gate/main.go b/cmd/release-gate/main.go index e14b4cbbe..8835f2916 100644 --- a/cmd/release-gate/main.go +++ b/cmd/release-gate/main.go @@ -46,6 +46,8 @@ func main() { ok, err = cmdMatrix(ctx, os.Args[2:]) case "invariants": ok, err = cmdInvariants(ctx, os.Args[2:]) + case "consistency": + ok, err = cmdConsistency(ctx, os.Args[2:]) case "run-suite": ok, err = cmdRunSuite(ctx, os.Args[2:]) case "report": diff --git a/docs/development/release-gate.md b/docs/development/release-gate.md index cf9662544..6945bd26d 100644 --- a/docs/development/release-gate.md +++ b/docs/development/release-gate.md @@ -33,6 +33,7 @@ fragments against a hardcoded manifest and exits per the verdict. | `suite-race` | `suite/unit-race`, `suite/server-race` | 25 min | `go test -race ./internal/...` + `go test -tags server -race ./internal/serveredition/...`. | | `suite-scan-eval` | `suite/scan-eval` | 10 min | `go run ./cmd/scan-eval --gate --min-recall 0.90 --max-fp 0.05` over the detect corpus — runs on **every** tag regardless of changed paths (FR-015). | | `matrix-invariants` | `matrix/{stdio,http,sse,docker,oauth}`, `invariant/{activity-request-id,counters,quarantine-flow,upgrade-in-place}` | 20 min | Boots the candidate against five local fixture upstreams (connect → list → call → kill/reconnect) and asserts the US2 invariants against the live instance. | +| `surface-consistency` | `invariant/surface-consistency` (T4) | 20 min | Boots the same matrix, adds quarantined/disabled/unhealthy/live servers and asserts the CLI, MCP and rendered Web UI agree with the core on every server's state — see [Surface consistency](#surface-consistency-t4--blocking). | | `web-ui-sweep` | — (advisory `advisory/web-ui-sweep`, T2) | 20 min | Playwright sweep over the Web UI **served by the candidate binary**, with a live stdio fixture upstream. `continue-on-error: true` — see [Web UI sweep](#web-ui-sweep-t2--advisory). | | `verdict` | (merges all) | 10 min | `release-gate report` → `gate-report.json` + `$GITHUB_STEP_SUMMARY`; its exit code **is** the gate verdict. | @@ -172,10 +173,56 @@ It is **advisory**, on purpose while the sweep earns its flake record: release tags with no flaky or infrastructure failure — the same criterion T3 uses (FR-021). -## Extension slots (T3 / T4) +## Surface consistency (T4) — blocking + +`invariant/surface-consistency` (job `surface-consistency`, US5 / FR-018) +proves the surfaces do not lie to each other. The job boots the same +five-type server matrix (stdio/http/sse/docker/oauth), then +`release-gate consistency` adds four servers so every admin state is on +screen — `gate-cons-live` (healthy), `gate-cons-quarantined`, +`gate-cons-disabled` and `gate-cons-broken` (endpoint refuses → unhealthy) — +and compares every surface against the core's `GET /api/v1/servers`: + +| Surface | Read path | Fields compared | +|---------|-----------|-----------------| +| CLI | `mcpproxy upstream list -o json --config ` (daemon client mode) | enabled, quarantined, connected, connection state, `health.level`, `health.admin_state`, `health.status`, `tool_count` | +| MCP | `/mcp` `upstream_servers` `operation=list` — storage + upstream manager + runtime health, an independent projection | enabled, quarantined, connection state, `health.level`, `health.admin_state`, `health.status` (the row carries no tool count) | +| Web UI | rendered server cards on `/ui/servers`, read by `e2e/web-ui-sweep/surface-snapshot.mjs` in one long-lived Chromium page | enabled (⋯ menu Enable/Disable), quarantined (⋯ Review), tool count, status chip colour and text — the expected chip is derived from the core row with `ServerCard.vue`'s rules, because the Web UI never prints `health.level` as text | +| macOS tray | — | not compared; recorded as `not_compared` in the fragment (T3) | + +Connection state is compared only for enabled, non-quarantined servers (a +disabled server is `idle` in the StateView and `disconnected` in the manager, +which no user sees). It runs three phases, each retried until the surfaces +agree or `--settle-timeout` (60s) runs out; an attempt counts only when the +core did not change while the surfaces were being read: + +1. **settled** — the mixed state, Web UI freshly loaded; +2. **live** — `gate-cons-live` is disabled through REST while the page stays + open, so the Web UI must follow through its SSE/poll update path; +3. **reload** — the same state after a page reload. Live failing while reload + passes is reported as a stale live view. + +A divergence that survives the retries fails the gate with one line per field: +`webui gate-cons-quarantined webui.chip_class: core=badge-secondary webui=badge-success`. +The fragment's `details` carry the last captured rows of every surface per +phase. + +**Run it by hand** next to `scripts/test-api-e2e.sh` (needs `make build` for +an embedded Web UI, Node for the reader): -The manifest reserves two non-blocking slots, recorded as -`not-run` / `not-implemented-yet` until their stage lands: +```bash +./scripts/test-surface-consistency.sh +``` + +**Prove it fails**: pass `--seed-mismatch surface:server:field=value` to the +script, or dispatch the gate with the `seed_consistency_mismatch` input (read +only on `workflow_dispatch`, never on a tag's `workflow_call`): + +```bash +./scripts/test-surface-consistency.sh --seed-mismatch cli:gate-cons-live:health.level=degraded +``` + +## Extension slot (T3) - **`reserved/macos-smoke` (T3)** — a macOS-runner job that launches the tray against a running core and uses the `mcpproxy-ui-test` accessibility @@ -185,10 +232,7 @@ The manifest reserves two non-blocking slots, recorded as added: *promote to blocking once it has passed on 3 consecutive release tags with no flaky or infrastructure failure.* Promotion is a one-line change (advisory → blocking on the manifest entry / job status handling). -- **`reserved/surface-consistency` (T4)** — one comparison over data the other - jobs already collect: REST vs CLI (`mcpproxy upstream list -o json`) vs Web UI - (vs tray, when T3 ran) agreement on each server's identity, admin state, and - health level, per the unified `health` contract (US5, FR-018). + When T3 lands, the tray joins the T4 comparison as one more surface. ## Runtime budget (FR-005 / SC-007) diff --git a/e2e/web-ui-sweep/surface-snapshot.mjs b/e2e/web-ui-sweep/surface-snapshot.mjs new file mode 100644 index 000000000..c2e67a845 --- /dev/null +++ b/e2e/web-ui-sweep/surface-snapshot.mjs @@ -0,0 +1,91 @@ +// Spec 081 T4 (FR-018) — Web UI reader for `release-gate consistency`. +// +// Not a Playwright test: a long-lived reader the Go driver talks to over +// stdin/stdout, one JSON line per command, so ONE page stays open across the +// check's phases (that is how the live phase tests the Web UI's update path +// rather than only its initial fetch). +// +// load open /ui/servers and wait for the server grid +// snapshot read the rendered cards as they are now (no reload) +// reload reload the page, then read the cards +// quit close the browser and exit +// +// Reply: {"ok":true,"rows":[{name, chip_class, chip_text, tool_count, +// enabled, quarantined}]} or {"ok":false,"error":"..."}. +// +// Env: MCPPROXY_BASE_URL, MCPPROXY_API_KEY (the instance under test), +// PW_CHROMIUM (optional explicit Chromium binary, as in the sweep config). +import { chromium } from '@playwright/test' +import readline from 'node:readline' + +const BASE = (process.env.MCPPROXY_BASE_URL || '').replace(/\/$/, '') +const KEY = process.env.MCPPROXY_API_KEY || '' +if (!BASE) { + process.stdout.write(JSON.stringify({ ok: false, error: 'MCPPROXY_BASE_URL is not set' }) + '\n') + process.exit(2) +} +const url = `${BASE}/ui/servers${KEY ? `?apikey=${encodeURIComponent(KEY)}` : ''}` + +const browser = await chromium.launch({ + headless: true, + ...(process.env.PW_CHROMIUM ? { executablePath: process.env.PW_CHROMIUM } : {}), +}) +const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }) + +async function waitGrid() { + await page.waitForSelector('[data-test="server-card"], [data-test="servers-first-run-empty"]', { timeout: 30_000 }) +} + +async function readCards() { + return page.$$eval('[data-test="server-card"]', (cards) => + cards.map((card) => { + const chip = card.querySelector('[data-test="server-status-chip"]') + const color = chip + ? [...chip.classList].find((c) => /^badge-(success|warning|error|neutral|secondary|ghost|info|accent|primary)$/.test(c)) || '' + : '' + const tools = card.querySelector('[data-test="server-card-tool-count"]') + const toolMatch = tools ? /(\d+)/.exec(tools.textContent || '') : null + const toggle = (card.querySelector('[data-test="server-card-menu-toggle"]')?.textContent || '').trim() + return { + name: card.querySelector('[data-test="server-card-title"]')?.getAttribute('title') || '', + chip_class: color, + chip_text: (chip?.textContent || '').trim(), + tool_count: toolMatch ? Number(toolMatch[1]) : null, + enabled: toggle === 'Disable' ? true : toggle === 'Enable' ? false : null, + quarantined: !!card.querySelector('[data-test="server-card-menu-review"]'), + } + }), + ) +} + +const handlers = { + async load() { + await page.goto(url) + await waitGrid() + return { ok: true, rows: await readCards() } + }, + async snapshot() { + return { ok: true, rows: await readCards() } + }, + async reload() { + await page.reload() + await waitGrid() + return { ok: true, rows: await readCards() } + }, +} + +const rl = readline.createInterface({ input: process.stdin }) +for await (const raw of rl) { + const cmd = raw.trim() + if (!cmd) continue + if (cmd === 'quit') break + let reply + try { + const h = handlers[cmd] + reply = h ? await h() : { ok: false, error: `unknown command ${cmd}` } + } catch (err) { + reply = { ok: false, error: String(err?.message || err) } + } + process.stdout.write(JSON.stringify(reply) + '\n') +} +await browser.close() diff --git a/internal/gatereport/gatereport.go b/internal/gatereport/gatereport.go index 38943b9d8..2cf934078 100644 --- a/internal/gatereport/gatereport.go +++ b/internal/gatereport/gatereport.go @@ -6,9 +6,8 @@ // Fragment file into a shared report directory. The merger compares the // fragments against a HARDCODED manifest of expected entries: a missing // fragment for a blocking entry is a FAIL (no silent skips, FR-004), and -// reserved extension slots (macos-smoke, surface-consistency — Spec 081 -// T3/T4) are recorded as `not-run` with reason "not-implemented-yet" until -// their stages land. The T2 Web UI sweep is a live ADVISORY entry: it runs on +// reserved extension slots (macos-smoke — Spec 081 T3) are recorded as +// `not-run` with reason "not-implemented-yet" until their stages land. The T2 Web UI sweep is a live ADVISORY entry: it runs on // tag builds and its failures are reported without blocking the release. package gatereport @@ -116,13 +115,19 @@ const ( // listed under advisory_failures) without blocking publication. EntryAdvisoryWebUISweep = "advisory/web-ui-sweep" - EntryReservedMacOSSmoke = "reserved/macos-smoke" - EntryReservedSurfaceConsistency = "reserved/surface-consistency" + EntryReservedMacOSSmoke = "reserved/macos-smoke" + + // EntrySurfaceConsistency is Spec 081 T4 (US5, FR-018): the CLI, MCP and + // Web UI surfaces agree with the core's /api/v1/servers on every server's + // state across the server-type matrix. Blocking. The macOS tray joins + // the comparison with T3; until then it is recorded as not-compared. + EntrySurfaceConsistency = "invariant/surface-consistency" ) // Manifest returns the hardcoded expected-entries manifest: 5 matrix cells + -// 4 invariants + 5 assembled suite jobs (FR-003) + 1 advisory entry (the T2 -// Web UI sweep) + 2 reserved slots (T3/T4). +// 5 invariants (incl. the T4 surface-consistency check) + 5 assembled suite +// jobs (FR-003) + 1 advisory entry (the T2 Web UI sweep) + 1 reserved slot +// (T3 macOS smoke). func Manifest() []ManifestEntry { return []ManifestEntry{ {Name: EntryMatrixStdio, Blocking: true}, @@ -135,6 +140,7 @@ func Manifest() []ManifestEntry { {Name: EntryInvariantCounters, Blocking: true}, {Name: EntryInvariantQuarantine, Blocking: true}, {Name: EntryInvariantUpgrade, Blocking: true}, + {Name: EntrySurfaceConsistency, Blocking: true}, {Name: EntrySuiteAPIE2E, Blocking: true}, {Name: EntrySuiteAPIE2ECleanupCheck, Blocking: true}, @@ -145,7 +151,6 @@ func Manifest() []ManifestEntry { {Name: EntryAdvisoryWebUISweep, Blocking: false}, {Name: EntryReservedMacOSSmoke, Blocking: false, Reserved: true}, - {Name: EntryReservedSurfaceConsistency, Blocking: false, Reserved: true}, } } diff --git a/internal/gatereport/gatereport_test.go b/internal/gatereport/gatereport_test.go index c40db1310..6d6fbe533 100644 --- a/internal/gatereport/gatereport_test.go +++ b/internal/gatereport/gatereport_test.go @@ -36,7 +36,7 @@ func TestMerge_AllBlockingPass_VerdictPass_ReservedNotRun(t *testing.T) { t.Fatalf("expected pass verdict, got %s (failures: %v)", r.Verdict, r.BlockingFailures) } // Reserved slots must be recorded, never silently absent (FR-004). - for _, name := range []string{EntryReservedMacOSSmoke, EntryReservedSurfaceConsistency} { + for _, name := range []string{EntryReservedMacOSSmoke} { e := entryByName(t, r, name) if e.Status != StatusNotRun || e.Reason != ReasonNotImplemented { t.Errorf("%s: got status=%s reason=%q, want not-run/%s", name, e.Status, e.Reason, ReasonNotImplemented) diff --git a/roadmap.yaml b/roadmap.yaml index 7565aec90..da1606b60 100644 --- a/roadmap.yaml +++ b/roadmap.yaml @@ -684,9 +684,10 @@ epics: depends_on: [release-qa-gate-matrix] - id: release-qa-gate-consistency title: "T4: surface-state consistency check (tray/Web UI/CLI agree with core on server states)" - status: todo + status: in_review priority: P3 depends_on: [release-qa-gate-matrix] + note: "BLOCKING gate entry invariant/surface-consistency (job surface-consistency): boots the five-type matrix, adds quarantined/disabled/unhealthy/live servers, and asserts CLI (`upstream list -o json`), MCP (`upstream_servers list`, an independent storage+manager projection) and the RENDERED Web UI (one long-lived Chromium page, e2e/web-ui-sweep/surface-snapshot.mjs) agree with core /api/v1/servers on enabled, quarantined, connection state, health.level/admin_state/status and tool_count. Three phases: settled, live (disable via REST, no page reload — tests the SSE/poll path), reload. Seeded negative: --seed-mismatch / workflow_dispatch seed_consistency_mismatch. Local: scripts/test-surface-consistency.sh. Tray recorded not-compared until T3." - id: telemetry-v7-churn title: "Telemetry v7: honest funnel + churn instrumentation" diff --git a/scripts/test-surface-consistency.sh b/scripts/test-surface-consistency.sh new file mode 100755 index 000000000..a38dff478 --- /dev/null +++ b/scripts/test-surface-consistency.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Spec 081 T4 — surface-state consistency check, runnable by hand next to +# scripts/test-api-e2e.sh. Same driver and same steps as the release gate's +# `surface-consistency` job (.github/workflows/release-qa-gate.yml): +# +# 1. `release-gate matrix` boots the candidate core against the server-type +# fixtures (stdio/http/sse/oauth, plus docker when Docker is available) +# and leaves it running; +# 2. `release-gate consistency` adds the mixed-state servers (quarantined, +# disabled, unhealthy, live) and asserts the CLI (`upstream list -o json`), +# the MCP `upstream_servers list` and the rendered Web UI agree with the +# core's /api/v1/servers on every server — settled, after a live change, +# and after a reload. Then it tears everything down. +# +# Usage: +# ./scripts/test-surface-consistency.sh # full check +# ./scripts/test-surface-consistency.sh --seed-mismatch cli:gate-cons-live:health.level=degraded +# # prove it fails +# ./scripts/test-surface-consistency.sh --no-webui # no Node/Chromium +# +# Extra arguments go to `release-gate consistency`. Environment: +# MCPPROXY_BINARY_PATH candidate core (default: ./mcpproxy; must embed the +# Web UI — build with `make build`) +# GATE_CELLS matrix cells (default: stdio,http,sse,oauth[,docker]) +# REPORT_DIR fragment output (default: tmp/surface-consistency) + +set -uo pipefail + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$REPO_ROOT" + +BIN_DIR="$REPO_ROOT/tmp/surface-consistency-bin" +REPORT_DIR="${REPORT_DIR:-$REPO_ROOT/tmp/surface-consistency}" +WORK_DIR=$(mktemp -d) +CORE="${MCPPROXY_BINARY_PATH:-$REPO_ROOT/mcpproxy}" +SWEEP_DIR="$REPO_ROOT/e2e/web-ui-sweep" +mkdir -p "$BIN_DIR" "$REPORT_DIR" +trap 'rm -rf "$WORK_DIR"' EXIT + +NO_WEBUI=0 +for arg in "$@"; do + [[ "$arg" == "--no-webui" ]] && NO_WEBUI=1 +done + +if [[ ! -x "$CORE" ]]; then + echo "candidate core not found at $CORE — run 'make build' (the Web UI must be embedded)" >&2 + exit 1 +fi + +echo "building gate driver + fixtures into $BIN_DIR" +go build -o "$BIN_DIR/release-gate" ./cmd/release-gate || exit 1 +go build -o "$BIN_DIR/mcpfixture" ./cmd/mcpfixture || exit 1 +go build -o "$BIN_DIR/oauthserver" ./tests/oauthserver/cmd/server || exit 1 + +CELLS="${GATE_CELLS:-}" +if [[ -z "$CELLS" ]]; then + CELLS="stdio,http,sse,oauth" + if docker info >/dev/null 2>&1; then + CELLS="$CELLS,docker" + else + echo "Docker unavailable — skipping the docker cell locally (the gate job always runs it)" + fi +fi +if [[ ",$CELLS," == *",docker,"* ]]; then + scripts/gate/build-fixture-image.sh || exit 1 +fi + +WEBUI_ARGS=() +if [[ $NO_WEBUI -eq 0 ]]; then + if [[ ! -x "$SWEEP_DIR/node_modules/.bin/playwright" ]]; then + (cd "$SWEEP_DIR" && npm ci) || exit 1 + fi + export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$REPO_ROOT/tmp/playwright-browsers}" + "$SWEEP_DIR/node_modules/.bin/playwright" install chromium || exit 1 + WEBUI_ARGS=(--webui-script "$SWEEP_DIR/surface-snapshot.mjs") +fi + +# The matrix writes its own fragments into a scratch dir: this script (and +# the gate job) only reports the consistency entry. +"$BIN_DIR/release-gate" matrix \ + --binary "$CORE" \ + --fixture "$BIN_DIR/mcpfixture" \ + --oauth-server "$BIN_DIR/oauthserver" \ + --cells "$CELLS" \ + --report-dir "$WORK_DIR/matrix-report" \ + --state-file "$WORK_DIR/gate-state.json" \ + --work-dir "$WORK_DIR" +echo "matrix exit=$? (its cells are reported by the matrix-invariants job, not here)" + +if [[ ! -f "$WORK_DIR/gate-state.json" ]]; then + echo "matrix left no state file — nothing to compare" >&2 + exit 1 +fi + +"$BIN_DIR/release-gate" consistency \ + --state-file "$WORK_DIR/gate-state.json" \ + --report-dir "$REPORT_DIR" \ + "${WEBUI_ARGS[@]}" \ + "$@" +RC=$? +echo "report fragment: $REPORT_DIR/invariant-surface-consistency.json" +exit $RC diff --git a/specs/081-release-qa-gate/tasks.md b/specs/081-release-qa-gate/tasks.md new file mode 100644 index 000000000..c04ac6fff --- /dev/null +++ b/specs/081-release-qa-gate/tasks.md @@ -0,0 +1,13 @@ +# Tasks: Release Qualification Gate (Spec 081) + +Mirrors the `release-qa-gate` epic in `roadmap.yaml`. Each task is one PR; +operating notes live in `docs/development/release-gate.md`. + +- [x] T1 — tag-blocking `release-qa-gate.yml`: server-type matrix (stdio/http/sse/docker/oauth), US2 invariants (activity-log request ids, counters, quarantine flow, upgrade-in-place), publish jobs in `release.yml`/`prerelease.yml` gated on the verdict, `scan-eval` unconditional on tags (US1, US2; FR-001–FR-015, FR-022) — #819 +- [x] T2 — Playwright Web UI sweep wired into the gate as `advisory/web-ui-sweep` (US3; FR-016/FR-017, advisory until promoted) — #1030 +- [ ] T3 — macOS tray smoke on a macOS runner via the `mcpproxy-ui-test` primitives, advisory until 3 consecutive passes (US4; FR-019–FR-021) +- [x] T4 — surface-state consistency: blocking `invariant/surface-consistency` job — CLI (`upstream list -o json`), MCP (`upstream_servers list`) and the rendered Web UI agree with the core's `/api/v1/servers` on enabled, quarantined, connection state, `health.level`/`admin_state`/`status` and tool counts across the five-type matrix plus quarantined/disabled/unhealthy servers; settled, live (no reload) and reload phases; per-field diff on failure; `--seed-mismatch` / `seed_consistency_mismatch` negative path; local `scripts/test-surface-consistency.sh`. Tray recorded as not-compared until T3 (US5; FR-018) + - [x] T4.1 — failing comparator tests first (`cmd/release-gate/consistency_test.go`, `consistency_workflow_test.go`) + - [x] T4.2 — `release-gate consistency` subcommand + Web UI reader `e2e/web-ui-sweep/surface-snapshot.mjs` + - [x] T4.3 — manifest: `reserved/surface-consistency` → blocking `invariant/surface-consistency`; gate job + verdict dependency + - [x] T4.4 — local runner `scripts/test-surface-consistency.sh`; docs From 54adbf1811140a9fc28e7c33f2e074c032623810 Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Sat, 17 Oct 2026 17:13:32 +0300 Subject: [PATCH 2/3] fix(release-gate): address round-1 review of the surface consistency check Comparator: - connection state stays compared for disabled/quarantined servers: exactly for the CLI, connected-vs-not for MCP - Web UI expectation mirrors ServerCard's no-health branch and the usable=false "Unavailable" fallback - --seed-mismatch refuses fields the surface is not compared on and servers the surface does not show Live command: - CLI child drops every MCPPROXY_* override (MCPPROXY_DATA picked a socket ahead of the gate's TCP endpoint) - every capture runs under the phase deadline; late agreement fails - a timed-out/cancelled Web UI command kills the reader, so a late reply is never read as the next command's answer - fragment written before teardown; teardown bounded to 2 minutes - stderr buffer is synchronized Local script: Bash 3.2 empty-array expansion, and a trap that runs the new `release-gate teardown` so an early failure leaks no processes. --- .github/workflows/release-qa-gate.yml | 2 +- cmd/release-gate/consistency.go | 134 +++++++++---- cmd/release-gate/consistency_cmd.go | 203 +++++++++++++++----- cmd/release-gate/consistency_reader_test.go | 32 +++ cmd/release-gate/consistency_test.go | 87 +++++++++ cmd/release-gate/main.go | 9 + scripts/test-surface-consistency.sh | 14 +- 7 files changed, 388 insertions(+), 93 deletions(-) create mode 100644 cmd/release-gate/consistency_reader_test.go diff --git a/.github/workflows/release-qa-gate.yml b/.github/workflows/release-qa-gate.yml index 4b08d536d..f45f7864f 100644 --- a/.github/workflows/release-qa-gate.yml +++ b/.github/workflows/release-qa-gate.yml @@ -495,7 +495,7 @@ jobs: --state-file "${WORK}/gate-state.json" \ --report-dir "${GATE_REPORT_DIR}" \ --webui-script "${GITHUB_WORKSPACE}/e2e/web-ui-sweep/surface-snapshot.mjs" \ - "${SEED_ARGS[@]}" + ${SEED_ARGS[@]+"${SEED_ARGS[@]}"} - name: Upload report fragment if: always() diff --git a/cmd/release-gate/consistency.go b/cmd/release-gate/consistency.go index 619022444..71cfc0d33 100644 --- a/cmd/release-gate/consistency.go +++ b/cmd/release-gate/consistency.go @@ -79,6 +79,9 @@ const ( hiddenSummary = "_health.summary" hiddenHealthAction = "_health.action" hiddenDiagnosticCode = "_diagnostic.code" + hiddenHasHealth = "_health.present" + hiddenUsable = "_health.usable" + hiddenConnecting = "_connecting" ) // surfaceFields lists what each surface is compared on. @@ -94,6 +97,7 @@ type rawServer struct { Enabled *bool `json:"enabled"` Quarantined *bool `json:"quarantined"` Connected *bool `json:"connected"` + Connecting *bool `json:"connecting"` Status string `json:"status"` ToolCount *int `json:"tool_count"` ConnectionStatus *struct { @@ -105,6 +109,7 @@ type rawServer struct { Status string `json:"status"` Summary string `json:"summary"` Action string `json:"action"` + Usable *bool `json:"usable"` } `json:"health"` Diagnostic *struct { Code string `json:"code"` @@ -124,6 +129,9 @@ func (s rawServer) row(connState string, withToolCount, withConnected bool) surf if withConnected && s.Connected != nil { r[fieldConnected] = strconv.FormatBool(*s.Connected) } + if s.Connecting != nil { + r[hiddenConnecting] = strconv.FormatBool(*s.Connecting) + } r[fieldConnection] = normalizeConnState(connState) if withToolCount && s.ToolCount != nil { r[fieldToolCount] = strconv.Itoa(*s.ToolCount) @@ -134,6 +142,10 @@ func (s rawServer) row(connState string, withToolCount, withConnected bool) surf r[fieldStatus] = s.Health.Status r[hiddenSummary] = s.Health.Summary r[hiddenHealthAction] = s.Health.Action + r[hiddenHasHealth] = "true" + if s.Health.Usable != nil { + r[hiddenUsable] = strconv.FormatBool(*s.Health.Usable) + } } if s.Diagnostic != nil { r[hiddenDiagnosticCode] = s.Diagnostic.Code @@ -257,46 +269,63 @@ var oauthReauthCodes = map[string]bool{ } // expectedWebUIRow derives what ServerCard.vue must render for a core row -// (statusBadgeClass / statusText; oauthSignInState from utils/health.ts). +// (statusBadgeClass / statusText; oauthSignInState and healthStatusText from +// utils/health.ts), including the no-health fallback branch. func expectedWebUIRow(core surfaceRow) surfaceRow { code := core[hiddenDiagnosticCode] signIn := oauthReauthCodes[code] || core[hiddenHealthAction] == health.ActionLogin || code == "MCPX_OAUTH_LOGIN_REQUIRED" + // admin_state only ever comes from the health object. + hasHealth := core[hiddenHasHealth] == "true" || core[fieldAdmin] != "" admin := core[fieldAdmin] + connected := core[fieldConnected] == "true" - var chip string - switch { - case admin == health.StateDisabled: - chip = "badge-neutral" - case signIn: - chip = "badge-warning" - case admin == health.StateQuarantined: - chip = "badge-secondary" - default: - switch core[fieldLevel] { - case health.LevelHealthy: - chip = "badge-success" - case health.LevelDegraded: + var chip, text string + if hasHealth { + switch { + case admin == health.StateDisabled: + chip = "badge-neutral" + case signIn: chip = "badge-warning" - case health.LevelUnhealthy: - chip = "badge-error" + case admin == health.StateQuarantined: + chip = "badge-secondary" default: - chip = "badge-ghost" + switch core[fieldLevel] { + case health.LevelHealthy: + chip = "badge-success" + case health.LevelDegraded: + chip = "badge-warning" + case health.LevelUnhealthy: + chip = "badge-error" + default: + chip = "badge-ghost" + } + } + switch { + case signIn && admin != health.StateDisabled: + text = health.StatusLabel(health.StatusSignInRequired) + case core[hiddenSummary] != "": + text = core[hiddenSummary] + case core[fieldStatus] != "": + text = health.StatusLabel(core[fieldStatus]) + case core[hiddenUsable] == "false" && connected: + text = "Unavailable" + case connected: + text = "Connected" + default: + text = "Disconnected" + } + } else { + switch { + case signIn: + chip, text = "badge-warning", health.StatusLabel(health.StatusSignInRequired) + case connected: + chip, text = "badge-success", "Connected" + case core[hiddenConnecting] == "true": + chip, text = "badge-warning", "Connecting" + default: + chip, text = "badge-error", "Disconnected" } - } - - var text string - switch { - case signIn && admin != health.StateDisabled: - text = health.StatusLabel(health.StatusSignInRequired) - case core[hiddenSummary] != "": - text = core[hiddenSummary] - case core[fieldStatus] != "": - text = health.StatusLabel(core[fieldStatus]) - case core[fieldConnected] == "true": - text = "Connected" - default: - text = "Disconnected" } return surfaceRow{ @@ -308,14 +337,25 @@ func expectedWebUIRow(core surfaceRow) surfaceRow { } } -// connectionStateMeaningful: a disabled or quarantined server has no live -// connection both projections are obliged to agree on ("Not Started" in the -// manager vs "disabled" in the StateView is not a user-visible divergence; -// the admin state and health fields cover those servers). +// connectionStateMeaningful: for an enabled, trusted server every surface +// must report the same connection state. A disabled or quarantined server +// has no live connection the projections share a word for ("Not Started" / +// "Disconnected" in the manager vs "idle" / "disabled" in the StateView), so +// for MCP such a row is compared as connected-vs-not instead (activeState). +// The CLI prints the core's own rows and is always compared exactly. func connectionStateMeaningful(core surfaceRow) bool { return core[fieldEnabled] == "true" && core[fieldQuarantined] == "false" } +// activeState buckets a normalized connection state: a surface that reports +// a disabled server as "ready" disagrees with one that reports it "idle". +func activeState(state string) string { + if state == "ready" || state == "connected" { + return "connected" + } + return "not-connected" +} + // compareSurfaces compares every surface snapshot against the core rows. // The result is sorted (surface, server, field) so diffs are stable. func compareSurfaces(core map[string]surfaceRow, snaps []surfaceSnapshot) []fieldMismatch { @@ -339,12 +379,13 @@ func compareSurfaces(core map[string]surfaceRow, snaps []surfaceSnapshot) []fiel want = expectedWebUIRow(coreRow) } for _, f := range fields { - if f == fieldConnection && !connectionStateMeaningful(coreRow) { - continue + w, g := want[f], got[f] + if f == fieldConnection && snap.Surface == surfaceMCP && !connectionStateMeaningful(coreRow) { + w, g = activeState(w), activeState(g) } - if want[f] != got[f] { + if w != g { out = append(out, fieldMismatch{Surface: snap.Surface, Server: name, Field: f, - Core: want[f], Got: got[f]}) + Core: w, Got: g}) } } } @@ -401,22 +442,31 @@ func parseSeedMismatch(s string) (*seedMismatch, error) { if len(fv) != 2 || fv[0] == "" || parts[1] == "" { return nil, fmt.Errorf("seed %q: want surface:server:field=value", s) } - if _, ok := surfaceFields[parts[0]]; !ok { + fields, ok := surfaceFields[parts[0]] + if !ok { return nil, fmt.Errorf("seed %q: unknown surface %q (cli, mcp, webui)", s, parts[0]) } + if !containsStr(fields, fv[0]) { + return nil, fmt.Errorf("seed %q: %s is not compared on %s (compared: %s)", s, fv[0], parts[0], strings.Join(fields, ", ")) + } return &seedMismatch{Surface: parts[0], Server: parts[1], Field: fv[0], Value: fv[1]}, nil } -func applySeed(seed *seedMismatch, snaps []surfaceSnapshot) { +// applySeed reports whether the seed landed; a seed for a server the surface +// does not show cannot change the verdict and must not pass silently. +func applySeed(seed *seedMismatch, snaps []surfaceSnapshot) bool { if seed == nil { - return + return false } + applied := false for i := range snaps { if snaps[i].Surface != seed.Surface || snaps[i].Rows == nil { continue } if row, ok := snaps[i].Rows[seed.Server]; ok { row[seed.Field] = seed.Value + applied = true } } + return applied } diff --git a/cmd/release-gate/consistency_cmd.go b/cmd/release-gate/consistency_cmd.go index 99b2a533d..aae90654d 100644 --- a/cmd/release-gate/consistency_cmd.go +++ b/cmd/release-gate/consistency_cmd.go @@ -15,6 +15,7 @@ import ( "path/filepath" "reflect" "strings" + "sync" "time" "github.com/smart-mcp-proxy/mcpproxy-go/internal/gatereport" @@ -77,6 +78,24 @@ func cmdConsistency(ctx context.Context, args []string) (bool, error) { return runConsistency(ctx, opts) } +// cmdTeardown stops the core, fixtures and containers a `matrix +// --state-file` run left running. Idempotent; used by cleanup traps. +func cmdTeardown(args []string) (bool, error) { + fs := flag.NewFlagSet("teardown", flag.ExitOnError) + stateFile := fs.String("state-file", "", "state file written by `matrix --state-file` (required)") + if err := fs.Parse(args); err != nil { + return false, err + } + st, err := readState(*stateFile) + if err != nil { + return false, err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + teardownState(ctx, st) + return true, nil +} + func runConsistency(ctx context.Context, opts consistencyOpts) (bool, error) { st, err := readState(opts.StateFile) if err != nil { @@ -88,13 +107,18 @@ func runConsistency(ctx context.Context, opts consistencyOpts) (bool, error) { frag := runTimed(gatereport.EntrySurfaceConsistency, func(frag *gatereport.Fragment) error { return checkSurfaceConsistency(ctx, st, opts, frag) }) + // Persist the verdict BEFORE teardown: a hung cleanup (an unresponsive + // Docker daemon) must not leave the blocking entry without a fragment. + writeErr := gatereport.WriteFragment(opts.ReportDir, frag) + logf("consistency: %s -> %s %s", frag.Name, frag.Status, frag.Reason) if !opts.KeepCore { - teardownState(ctx, st) + cctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + teardownState(cctx, st) + cancel() } - if err := gatereport.WriteFragment(opts.ReportDir, frag); err != nil { - return false, fmt.Errorf("write fragment: %w", err) + if writeErr != nil { + return false, fmt.Errorf("write fragment: %w", writeErr) } - logf("consistency: %s -> %s %s", frag.Name, frag.Status, frag.Reason) return frag.Status == gatereport.StatusPass, nil } @@ -326,7 +350,7 @@ func (c *capturer) cli(ctx context.Context) surfaceSnapshot { defer cancel() cmd := exec.CommandContext(cctx, c.cliBinary, "upstream", "list", "-o", "json", "--config", c.configPath) // Never let the caller's environment point the CLI at another daemon. - cmd.Env = filteredEnv(os.Environ(), "MCPPROXY_TRAY_ENDPOINT", "MCPPROXY_API_KEY", "MCPPROXY_OUTPUT", "MCPPROXY_LISTEN") + cmd.Env = cliChildEnv(os.Environ()) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr if err := cmd.Run(); err != nil { @@ -366,29 +390,53 @@ func (c *capturer) webui(ctx context.Context, cmd string) surfaceSnapshot { // converge captures every surface until they agree with the core or the // budget runs out. An attempt only counts when the core did not move while // the surfaces were being read (core-before == core-after) — otherwise a -// legitimate transition would read as a disagreement. The first Web UI read -// uses webCmd ("snapshot" or "reload"); later attempts only re-read the page. +// legitimate transition would read as a disagreement. Every capture runs +// under the phase deadline, and agreement observed after it does not count. +// The first Web UI read uses webCmd ("snapshot" or "reload"); later attempts +// only re-read the page. func (c *capturer) converge(ctx context.Context, webCmd string, budget time.Duration) ([]fieldMismatch, error) { deadline := time.Now().Add(budget) + pctx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() var last []fieldMismatch stableAttempts := 0 + expired := func() ([]fieldMismatch, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if stableAttempts == 0 { + return nil, fmt.Errorf("no complete, stable capture of every surface within %s", budget) + } + return last, nil + } for attempt := 1; ; attempt++ { - before, err := c.core(ctx) + before, err := c.core(pctx) if err != nil { + if pctx.Err() != nil { + return expired() + } return nil, &infraError{fmt.Errorf("read core: %w", err)} } - snaps := []surfaceSnapshot{c.cli(ctx), c.mcpList(ctx)} + snaps := []surfaceSnapshot{c.cli(pctx), c.mcpList(pctx)} if c.web != nil { - snaps = append(snaps, c.webui(ctx, webCmd)) + snaps = append(snaps, c.webui(pctx, webCmd)) webCmd = "snapshot" } - after, err := c.core(ctx) + after, err := c.core(pctx) if err != nil { + if pctx.Err() != nil { + return expired() + } return nil, &infraError{fmt.Errorf("read core: %w", err)} } + if time.Now().After(deadline) { + return expired() + } if reflect.DeepEqual(before, after) { stableAttempts++ - applySeed(c.seed, snaps) + if c.seed != nil && !applySeed(c.seed, snaps) { + return nil, fmt.Errorf("--seed-mismatch: surface %s does not show server %q, the seed cannot take effect", c.seed.Surface, c.seed.Server) + } c.lastCore, c.lastSnaps = after, snaps last = compareSurfaces(after, snaps) if len(last) == 0 { @@ -399,45 +447,62 @@ func (c *capturer) converge(ctx context.Context, webCmd string, budget time.Dura } else { logf("consistency: attempt %d: core state moved during capture, retrying", attempt) } - if time.Now().After(deadline) { - if stableAttempts == 0 { - return nil, fmt.Errorf("core state never held still for one capture within %s", budget) - } - return last, nil - } select { - case <-ctx.Done(): - return nil, ctx.Err() + case <-pctx.Done(): + return expired() case <-time.After(3 * time.Second): } } } -func filteredEnv(env []string, drop ...string) []string { +// cliChildEnv strips every MCPPROXY_* variable from the CLI's environment: +// MCPPROXY_DATA, MCPPROXY_TRAY_ENDPOINT, MCPPROXY_API_KEY, MCPPROXY_LISTEN… +// can each point the CLI at another daemon than the gate's --config names. +func cliChildEnv(env []string) []string { out := make([]string, 0, len(env)) for _, kv := range env { - keep := true - for _, d := range drop { - if strings.HasPrefix(kv, d+"=") { - keep = false - break - } - } - if keep { + if !strings.HasPrefix(kv, "MCPPROXY_") { out = append(out, kv) } } return out } +// lockedBuffer is a bytes.Buffer safe for os/exec's stderr copier and our +// error paths to use concurrently. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // webuiReader drives e2e/web-ui-sweep/surface-snapshot.mjs: one long-lived // headless Chromium page, one JSON line per command on stdin/stdout. Keeping // the page open is what lets the live phase test the Web UI's update path. +// +// Replies are not tagged, so a command that times out or is cancelled kills +// the reader for good: a late reply must never be read as the answer to a +// later command. type webuiReader struct { - cmd *exec.Cmd - stdin io.WriteCloser - lines chan []byte - stderr *bytes.Buffer + cmd *exec.Cmd + stdin io.WriteCloser + lines chan []byte + stderr *lockedBuffer + done chan struct{} + once sync.Once + dead error + timeout time.Duration } func startWebUIReader(ctx context.Context, script, baseURL, apiKey string) (*webuiReader, error) { @@ -446,8 +511,11 @@ func startWebUIReader(ctx context.Context, script, baseURL, apiKey string) (*web } cmd := exec.CommandContext(ctx, "node", script) cmd.Dir = filepath.Dir(script) // resolves @playwright/test from the sweep's node_modules - cmd.Env = append(filteredEnv(os.Environ(), "MCPPROXY_API_KEY", "MCPPROXY_BASE_URL"), - "MCPPROXY_BASE_URL="+baseURL, "MCPPROXY_API_KEY="+apiKey) + cmd.Env = append(cliChildEnv(os.Environ()), "MCPPROXY_BASE_URL="+baseURL, "MCPPROXY_API_KEY="+apiKey) + return startReaderCmd(cmd) +} + +func startReaderCmd(cmd *exec.Cmd) (*webuiReader, error) { stdin, err := cmd.StdinPipe() if err != nil { return nil, err @@ -456,48 +524,87 @@ func startWebUIReader(ctx context.Context, script, baseURL, apiKey string) (*web if err != nil { return nil, err } - stderr := &bytes.Buffer{} - cmd.Stderr = stderr + r := &webuiReader{stdin: stdin, lines: make(chan []byte, 4), stderr: &lockedBuffer{}, + done: make(chan struct{}), timeout: 90 * time.Second} + cmd.Stderr = r.stderr + r.cmd = cmd if err := cmd.Start(); err != nil { return nil, err } - r := &webuiReader{cmd: cmd, stdin: stdin, lines: make(chan []byte, 4), stderr: stderr} go func() { + defer close(r.lines) sc := bufio.NewScanner(stdout) sc.Buffer(make([]byte, 1<<20), 16<<20) for sc.Scan() { - r.lines <- append([]byte(nil), sc.Bytes()...) + select { + case r.lines <- append([]byte(nil), sc.Bytes()...): + case <-r.done: + return + } } - close(r.lines) }() return r, nil } +// kill stops the reader for good; later commands fail with reason. +func (r *webuiReader) kill(reason error) { + r.once.Do(func() { + r.dead = reason + close(r.done) + _ = r.stdin.Close() + if r.cmd.Process != nil { + _ = r.cmd.Process.Kill() + } + go func() { _ = r.cmd.Wait() }() + }) +} + func (r *webuiReader) do(ctx context.Context, command string) ([]byte, error) { + select { + case <-r.done: + return nil, fmt.Errorf("web UI reader is gone: %w", r.dead) + default: + } if _, err := io.WriteString(r.stdin, command+"\n"); err != nil { - return nil, fmt.Errorf("send %q: %w (stderr: %s)", command, err, truncateStr(r.stderr.String(), 400)) + err = fmt.Errorf("send %q: %w (stderr: %s)", command, err, truncateStr(r.stderr.String(), 400)) + r.kill(err) + return nil, err } + timer := time.NewTimer(r.timeout) + defer timer.Stop() select { case line, ok := <-r.lines: if !ok { - return nil, fmt.Errorf("web UI reader exited (stderr: %s)", truncateStr(r.stderr.String(), 600)) + err := fmt.Errorf("web UI reader exited (stderr: %s)", truncateStr(r.stderr.String(), 600)) + r.kill(err) + return nil, err } return line, nil - case <-time.After(90 * time.Second): - return nil, fmt.Errorf("web UI reader: %q timed out", command) + case <-timer.C: + err := fmt.Errorf("web UI reader: %q timed out after %s", command, r.timeout) + r.kill(err) + return nil, err case <-ctx.Done(): - return nil, ctx.Err() + err := fmt.Errorf("web UI reader: %q: %w", command, ctx.Err()) + r.kill(err) + return nil, err } } func (r *webuiReader) close() { + select { + case <-r.done: + return + default: + } _, _ = io.WriteString(r.stdin, "quit\n") _ = r.stdin.Close() - done := make(chan struct{}) - go func() { _ = r.cmd.Wait(); close(done) }() + exited := make(chan struct{}) + go func() { _ = r.cmd.Wait(); close(exited) }() select { - case <-done: + case <-exited: case <-time.After(10 * time.Second): _ = r.cmd.Process.Kill() } + r.once.Do(func() { r.dead = errors.New("closed"); close(r.done) }) } diff --git a/cmd/release-gate/consistency_reader_test.go b/cmd/release-gate/consistency_reader_test.go new file mode 100644 index 000000000..5054ce271 --- /dev/null +++ b/cmd/release-gate/consistency_reader_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "os/exec" + "strings" + "testing" + "time" +) + +// Sol 6.1 round 1 (livecmd finding 3): after a timed-out command the reader +// is dead — a late reply must never be taken as the answer to a later +// command (the reload phase would otherwise read the stale live snapshot). +func TestWebUIReader_TimeoutIsTerminal(t *testing.T) { + ctx := context.Background() + // Answers each command after 400ms — longer than the reader's timeout. + cmd := exec.CommandContext(ctx, "sh", "-c", `while read l; do sleep 0.4; echo '{"ok":true,"rows":[]}'; done`) + r, err := startReaderCmd(cmd) + if err != nil { + t.Fatalf("start: %v", err) + } + defer r.close() + r.timeout = 100 * time.Millisecond + if _, err := r.do(ctx, "snapshot"); err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("first command: err=%v, want timeout", err) + } + time.Sleep(600 * time.Millisecond) // the late reply has arrived by now + r.timeout = 2 * time.Second + if line, err := r.do(ctx, "reload"); err == nil { + t.Fatalf("dead reader answered %q", line) + } +} diff --git a/cmd/release-gate/consistency_test.go b/cmd/release-gate/consistency_test.go index 06e3f89b0..e2c0eae1f 100644 --- a/cmd/release-gate/consistency_test.go +++ b/cmd/release-gate/consistency_test.go @@ -218,3 +218,90 @@ func TestMismatchReportIsSortedAndNamesBothValues(t *testing.T) { t.Fatalf("formatted diff = %q", s) } } + +// Sol 6.1 round 1, finding 1: connection state stays compared for disabled +// and quarantined servers — exactly for the CLI (same projection as core), +// as connected-vs-not for MCP (whose vocabulary for an inactive server +// differs: "Not Started"/"Disconnected" vs the StateView's "idle"/"disabled"). +func TestCompareSurfaces_InactiveServerConnectionState(t *testing.T) { + core, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + if mm := compareSurfaces(core, snaps); len(mm) != 0 { + t.Fatalf("vocabulary-only difference flagged: %+v", mm) + } + + badCLI := strings.Replace(cliFixture, `"status":"disabled"`, `"status":"Ready"`, 1) + core, snaps = mustSnapshots(t, badCLI, mcpFixture, webuiFixture) + mm := compareSurfaces(core, snaps) + if len(mm) != 1 || mm[0].Surface != surfaceCLI || mm[0].Server != "off" || mm[0].Field != fieldConnection { + t.Fatalf("CLI disabled-server state divergence: got %+v", mm) + } + + badMCP := strings.Replace(mcpFixture, `"state":"Not Started"`, `"state":"Ready"`, 1) + core, snaps = mustSnapshots(t, cliFixture, badMCP, webuiFixture) + mm = compareSurfaces(core, snaps) + if len(mm) != 1 || mm[0].Surface != surfaceMCP || mm[0].Server != "off" || mm[0].Field != fieldConnection { + t.Fatalf("MCP disabled server reported Ready: got %+v", mm) + } +} + +// Sol 6.1 round 1, finding 2: the Web UI expectation mirrors ServerCard.vue's +// no-health branch and healthStatusText's usable=false fallback. +func TestExpectedWebUIRow_Fallbacks(t *testing.T) { + cases := []struct { + name string + row surfaceRow + chip, chipText string + }{ + {"no health, connected", surfaceRow{fieldConnected: "true"}, "badge-success", "Connected"}, + {"no health, connecting", surfaceRow{fieldConnected: "false", hiddenConnecting: "true"}, "badge-warning", "Connecting"}, + {"no health, down", surfaceRow{fieldConnected: "false"}, "badge-error", "Disconnected"}, + {"no health, sign-in", surfaceRow{fieldConnected: "false", hiddenDiagnosticCode: "MCPX_OAUTH_LOGIN_REQUIRED"}, "badge-warning", "Sign-in required"}, + {"health, unusable but connected", surfaceRow{hiddenHasHealth: "true", fieldAdmin: "enabled", fieldLevel: "degraded", + fieldConnected: "true", hiddenUsable: "false"}, "badge-warning", "Unavailable"}, + {"health, empty text, connected", surfaceRow{hiddenHasHealth: "true", fieldAdmin: "enabled", fieldLevel: "healthy", + fieldConnected: "true", hiddenUsable: "true"}, "badge-success", "Connected"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := expectedWebUIRow(tc.row) + if got[fieldChipClass] != tc.chip || got[fieldChipText] != tc.chipText { + t.Fatalf("got chip=%q text=%q, want %q %q", got[fieldChipClass], got[fieldChipText], tc.chip, tc.chipText) + } + }) + } +} + +// Sol 6.1 round 1, finding 3: a seed that cannot change the verdict is +// refused rather than silently passing the "prove it fails" run. +func TestSeedMismatch_RefusesNoOpSeeds(t *testing.T) { + if _, err := parseSeedMismatch("cli:x:health.leveel=degraded"); err == nil { + t.Fatal("a field the surface is not compared on was accepted") + } + if _, err := parseSeedMismatch("mcp:x:tool_count=9"); err == nil { + t.Fatal("tool_count is not compared for mcp, seed must be refused") + } + _, snaps := mustSnapshots(t, cliFixture, mcpFixture, webuiFixture) + seed, _ := parseSeedMismatch("cli:no-such-server:health.level=degraded") + if applySeed(seed, snaps) { + t.Fatal("seed for an absent server reported as applied") + } + seed, _ = parseSeedMismatch("cli:stdio-ok:health.level=degraded") + if !applySeed(seed, snaps) { + t.Fatal("valid seed not applied") + } +} + +// Sol 6.1 round 1 (livecmd finding 1): the CLI child must not inherit any +// MCPPROXY_* override that could point it at another daemon (MCPPROXY_DATA +// selects a socket ahead of the configured TCP endpoint). +func TestCLIEnvDropsEveryMCPProxyOverride(t *testing.T) { + env := cliChildEnv([]string{"PATH=/bin", "MCPPROXY_DATA=/home/u/.mcpproxy", "MCPPROXY_TRAY_ENDPOINT=x", "HOME=/h"}) + for _, kv := range env { + if strings.HasPrefix(kv, "MCPPROXY_") { + t.Fatalf("CLI env kept %s", kv) + } + } + if len(env) != 2 { + t.Fatalf("non-mcpproxy vars dropped: %v", env) + } +} diff --git a/cmd/release-gate/main.go b/cmd/release-gate/main.go index 8835f2916..b2d3fb0ee 100644 --- a/cmd/release-gate/main.go +++ b/cmd/release-gate/main.go @@ -9,6 +9,10 @@ // assert FR-011 (activity request-id completeness), FR-012 // (counter movement), FR-013 (quarantine + approval e2e) and // FR-014 (upgrade-in-place from the latest stable release). +// consistency attach to the live instance the matrix left running and +// assert the CLI, MCP and Web UI agree with the core on every +// server's state (T4, FR-018). +// teardown stop everything a matrix --state-file run left running. // run-suite run an existing suite entry point (FR-003) and record its // outcome as a report fragment. // report merge all fragments against the hardcoded manifest into @@ -48,6 +52,8 @@ func main() { ok, err = cmdInvariants(ctx, os.Args[2:]) case "consistency": ok, err = cmdConsistency(ctx, os.Args[2:]) + case "teardown": + ok, err = cmdTeardown(os.Args[2:]) case "run-suite": ok, err = cmdRunSuite(ctx, os.Args[2:]) case "report": @@ -78,6 +84,9 @@ usage: [--work-dir DIR] [--cell-timeout 300s] release-gate invariants --state-file FILE --report-dir DIR [--prev-binary PATH] [--skip-upgrade] \ [--upgrade-repo owner/repo] [--keep-core] + release-gate consistency --state-file FILE --report-dir DIR (--webui-script surface-snapshot.mjs | --no-webui) \ + [--seed-mismatch surface:server:field=value] [--settle-timeout 60s] [--keep-core] + release-gate teardown --state-file FILE release-gate run-suite --name suite/api-e2e --report-dir DIR [--advisory] -- CMD [ARGS...] release-gate report --report-dir DIR [--out gate-report.json] [--summary summary.md] `) diff --git a/scripts/test-surface-consistency.sh b/scripts/test-surface-consistency.sh index a38dff478..0e5f06ff3 100755 --- a/scripts/test-surface-consistency.sh +++ b/scripts/test-surface-consistency.sh @@ -36,7 +36,17 @@ WORK_DIR=$(mktemp -d) CORE="${MCPPROXY_BINARY_PATH:-$REPO_ROOT/mcpproxy}" SWEEP_DIR="$REPO_ROOT/e2e/web-ui-sweep" mkdir -p "$BIN_DIR" "$REPORT_DIR" -trap 'rm -rf "$WORK_DIR"' EXIT +cleanup() { + # The matrix leaves its core + fixtures running for the consistency step, + # which tears them down itself — unless it never got that far (bad flags, + # an early error). Teardown is idempotent, so always run it before the + # state file it needs is deleted. + if [[ -f "$WORK_DIR/gate-state.json" && -x "$BIN_DIR/release-gate" ]]; then + "$BIN_DIR/release-gate" teardown --state-file "$WORK_DIR/gate-state.json" >/dev/null 2>&1 || true + fi + rm -rf "$WORK_DIR" +} +trap cleanup EXIT NO_WEBUI=0 for arg in "$@"; do @@ -96,7 +106,7 @@ fi "$BIN_DIR/release-gate" consistency \ --state-file "$WORK_DIR/gate-state.json" \ --report-dir "$REPORT_DIR" \ - "${WEBUI_ARGS[@]}" \ + ${WEBUI_ARGS[@]+"${WEBUI_ARGS[@]}"} \ "$@" RC=$? echo "report fragment: $REPORT_DIR/invariant-surface-consistency.json" From 03ad256c9a4766ce2a8b0af55e2010275f7eac9f Mon Sep 17 00:00:00 2001 From: Algis Dumbris Date: Sat, 17 Oct 2026 17:16:58 +0300 Subject: [PATCH 3/3] fix(release-gate): bound the whole teardown; honour --keep-core in the local script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 review: teardown bounded Docker calls only (PID loops and pgrep take no context), so it now runs in a goroutine abandoned after its budget — the verdict is already written. The local script's new cleanup trap tore down an instance kept with --keep-core; it now leaves it up and prints the teardown command. --- cmd/release-gate/consistency_cmd.go | 30 ++++++++++++++++----- cmd/release-gate/consistency_reader_test.go | 12 +++++++++ scripts/test-surface-consistency.sh | 15 +++++++++-- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/cmd/release-gate/consistency_cmd.go b/cmd/release-gate/consistency_cmd.go index aae90654d..cc16ff63d 100644 --- a/cmd/release-gate/consistency_cmd.go +++ b/cmd/release-gate/consistency_cmd.go @@ -90,10 +90,30 @@ func cmdTeardown(args []string) (bool, error) { if err != nil { return false, err } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + return boundedTeardown(st, teardownBudget), nil +} + +// teardownBudget caps cleanup. Docker calls take the context, but the PID +// loops and pgrep do not, so the whole teardown runs in a goroutine and is +// abandoned (logged) once the budget is spent — the verdict is already on +// disk by then. +const teardownBudget = 2 * time.Minute + +func boundedTeardown(st *gateState, budget time.Duration) bool { + ctx, cancel := context.WithTimeout(context.Background(), budget) defer cancel() - teardownState(ctx, st) - return true, nil + done := make(chan struct{}) + go func() { + teardownState(ctx, st) + close(done) + }() + select { + case <-done: + return true + case <-time.After(budget + 10*time.Second): + logf("teardown: still running after %s — abandoning it (core pid %d)", budget, st.CorePID) + return false + } } func runConsistency(ctx context.Context, opts consistencyOpts) (bool, error) { @@ -112,9 +132,7 @@ func runConsistency(ctx context.Context, opts consistencyOpts) (bool, error) { writeErr := gatereport.WriteFragment(opts.ReportDir, frag) logf("consistency: %s -> %s %s", frag.Name, frag.Status, frag.Reason) if !opts.KeepCore { - cctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - teardownState(cctx, st) - cancel() + boundedTeardown(st, teardownBudget) } if writeErr != nil { return false, fmt.Errorf("write fragment: %w", writeErr) diff --git a/cmd/release-gate/consistency_reader_test.go b/cmd/release-gate/consistency_reader_test.go index 5054ce271..d74d4f34e 100644 --- a/cmd/release-gate/consistency_reader_test.go +++ b/cmd/release-gate/consistency_reader_test.go @@ -30,3 +30,15 @@ func TestWebUIReader_TimeoutIsTerminal(t *testing.T) { t.Fatalf("dead reader answered %q", line) } } + +// Sol 6.1 round 2: teardown is bounded as a whole, not just its Docker calls. +func TestBoundedTeardownReturnsOnTime(t *testing.T) { + start := time.Now() + // A state with nothing to stop finishes immediately. + if !boundedTeardown(&gateState{}, time.Second) { + t.Fatal("empty teardown reported as abandoned") + } + if time.Since(start) > 5*time.Second { + t.Fatalf("empty teardown took %s", time.Since(start)) + } +} diff --git a/scripts/test-surface-consistency.sh b/scripts/test-surface-consistency.sh index 0e5f06ff3..e397f6889 100755 --- a/scripts/test-surface-consistency.sh +++ b/scripts/test-surface-consistency.sh @@ -36,11 +36,22 @@ WORK_DIR=$(mktemp -d) CORE="${MCPPROXY_BINARY_PATH:-$REPO_ROOT/mcpproxy}" SWEEP_DIR="$REPO_ROOT/e2e/web-ui-sweep" mkdir -p "$BIN_DIR" "$REPORT_DIR" +KEEP_CORE=0 +for arg in "$@"; do + [[ "$arg" == "--keep-core" ]] && KEEP_CORE=1 +done + cleanup() { # The matrix leaves its core + fixtures running for the consistency step, # which tears them down itself — unless it never got that far (bad flags, - # an early error). Teardown is idempotent, so always run it before the - # state file it needs is deleted. + # an early error). Teardown is idempotent, so run it before the state file + # it needs is deleted. --keep-core means "leave it all up": keep the state + # file too, so `release-gate teardown` can stop it later. + if [[ $KEEP_CORE -eq 1 && -f "$WORK_DIR/gate-state.json" ]]; then + echo "--keep-core: instance left running; stop it with:" + echo " $BIN_DIR/release-gate teardown --state-file $WORK_DIR/gate-state.json" + return + fi if [[ -f "$WORK_DIR/gate-state.json" && -x "$BIN_DIR/release-gate" ]]; then "$BIN_DIR/release-gate" teardown --state-file "$WORK_DIR/gate-state.json" >/dev/null 2>&1 || true fi