diff --git a/cmd/mcpproxy/upstream_cmd.go b/cmd/mcpproxy/upstream_cmd.go index 506867659..cd52c46f7 100644 --- a/cmd/mcpproxy/upstream_cmd.go +++ b/cmd/mcpproxy/upstream_cmd.go @@ -23,6 +23,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/configimport" "github.com/smart-mcp-proxy/mcpproxy-go/internal/health" "github.com/smart-mcp-proxy/mcpproxy-go/internal/logs" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/reqcontext" ) @@ -1783,14 +1784,21 @@ func outputImportResultStructured(result *configimport.ImportResult, format stri // buildImportedServersOutput builds the output structure for imported servers func buildImportedServersOutput(imported []*configimport.ImportedServer) []map[string]interface{} { + // Issue #1148, round 8: the import preview echoes the operator's source + // file back as CLI JSON — a shape an agent pipes as readily as a human + // reads. The shared LIVE rule leaves an ordinary command, path or URL + // byte-identical and renders a credential as `•••• ( chars)`, so + // the operator can still verify that the secret imported and how long it + // is without the value itself reaching stdout. result := make([]map[string]interface{}, len(imported)) for i, s := range imported { + view := oauth.RedactedConfigView("", s.Server) result[i] = map[string]interface{}{ "name": s.Server.Name, "protocol": s.Server.Protocol, - "url": s.Server.URL, - "command": s.Server.Command, - "args": s.Server.Args, + "url": viewOr(view, "url", s.Server.URL), + "command": viewOr(view, "command", s.Server.Command), + "args": oauth.LiveRedaction.Argv(s.Server.Args), "enabled": s.Server.Enabled, "quarantined": s.Server.Quarantined, "source_format": s.SourceFormat, @@ -2285,3 +2293,14 @@ func runUpstreamPatch(_ *cobra.Command, args []string) error { fmt.Println() return nil } + +// viewOr reads one string leaf out of a redacted server view (see +// oauth.RedactedConfigView), falling back to the raw value when the key was +// omitted — every string field of config.ServerConfig is `omitempty`, and an +// empty value carries no secret. +func viewOr(view map[string]interface{}, key, fallback string) string { + if v, ok := view[key].(string); ok { + return v + } + return fallback +} diff --git a/internal/httpapi/diagnostics_per_server.go b/internal/httpapi/diagnostics_per_server.go index 076951d09..cc3c2d4ce 100644 --- a/internal/httpapi/diagnostics_per_server.go +++ b/internal/httpapi/diagnostics_per_server.go @@ -16,6 +16,7 @@ import ( "github.com/go-chi/chi/v5" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" "github.com/smart-mcp-proxy/mcpproxy-go/internal/diagnostics" "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" @@ -34,7 +35,11 @@ func redactHealthDetail(healthRaw interface{}, reveal bool) interface{} { return healthRaw } clone := *hs - clone.Detail = oauth.RedactSensitiveData(clone.Detail) + // Round 8 finding 2: oauth.ScrubUpstreamText is the ONE rule for + // free-form upstream text. RedactSensitiveData alone is its name half, and + // oauth.RedactServerSecretFields scrubs this very field with both halves on + // the sibling /api/v1/servers door. + clone.Detail = oauth.ScrubUpstreamText(clone.Detail) return &clone } @@ -45,7 +50,8 @@ func redactDiagnosticCause(diag map[string]interface{}, reveal bool) { return } if cause, ok := diag["cause"].(string); ok && cause != "" { - diag["cause"] = oauth.RedactSensitiveData(cause) + // Round 8 finding 2: the same one rule the sibling REST door applies. + diag["cause"] = oauth.ScrubUpstreamText(cause) } } @@ -127,3 +133,37 @@ func (s *Server) handleGetServerDiagnostics(w http.ResponseWriter, r *http.Reque s.writeSuccess(w, resp) } + +// viewString reads one string leaf out of a redacted server view (see +// oauth.RedactedConfigView), falling back to the raw value when the key was +// omitted — every string field of config.ServerConfig is `omitempty`, and an +// empty value carries no secret. It is the httpapi twin of the helper the MCP +// door uses, so the two build their echoes from the view the same way. +func viewString(view map[string]interface{}, key, fallback string) string { + if v, ok := view[key].(string); ok { + return v + } + return fallback +} + +// redactedRegistrySummary renders one registry source for a REST echo with its +// URLs masked by the shared LIVE rule. +// +// Issue #1148, round 8: a CUSTOM registry source is operator-configured, so its +// URL can carry a credential in the query string exactly as an upstream URL +// can — and it was echoed verbatim by add-source / edit-source / remove-source +// and republished by `list_registries` on every surface. The write doors +// (Server.AddRegistrySource / EditRegistrySource) refuse an echoed mask rather +// than persisting it over the credential, which is the same bind-or-refuse +// answer the server write path gives. +func redactedRegistrySummary(entry *config.RegistryEntry) contracts.RegistrySummary { + return contracts.RegistrySummary{ + ID: entry.ID, + Name: entry.Name, + URL: oauth.LiveRedaction.URLValue(entry.URL), + ServersURL: oauth.LiveRedaction.URLValue(entry.ServersURL), + Protocol: entry.Protocol, + Provenance: entry.Provenance, + Trusted: entry.IsTrusted(), + } +} diff --git a/internal/httpapi/import.go b/internal/httpapi/import.go index 7778d8a17..c866210ee 100644 --- a/internal/httpapi/import.go +++ b/internal/httpapi/import.go @@ -12,6 +12,7 @@ import ( "time" "github.com/smart-mcp-proxy/mcpproxy-go/internal/configimport" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" ) // ImportRequest represents a request to import servers from JSON/TOML content @@ -427,12 +428,23 @@ func (s *Server) runImport(r *http.Request, content []byte, formatHint string, s } for i, imported := range result.Imported { + // Issue #1148, round 9: this preview echoes the operator's SOURCE FILE + // back — an agent pipes it as readily as a human reads it — and it did + // so with `url`, `command` and `args` raw, while its CLI twin + // (`upstream import`, buildImportedServersOutput) was routed through + // the shared LIVE rule in round 8. Same data, two doors, one redacted. + // + // The shared rule leaves an ordinary command, path or flag + // byte-identical and renders a credential as `•••• ( chars)`, + // so the operator can still verify that a secret imported and how long + // it is without the value itself reaching the wire. + view := oauth.RedactedConfigView("", imported.Server) response.Imported[i] = ImportedServerResponse{ Name: imported.Server.Name, Protocol: imported.Server.Protocol, - URL: imported.Server.URL, - Command: imported.Server.Command, - Args: imported.Server.Args, + URL: viewString(view, "url", imported.Server.URL), + Command: viewString(view, "command", imported.Server.Command), + Args: oauth.LiveRedaction.Argv(imported.Server.Args), SourceFormat: string(imported.SourceFormat), OriginalName: imported.OriginalName, FieldsSkipped: imported.FieldsSkipped, diff --git a/internal/httpapi/round8_diagnostics_rule_test.go b/internal/httpapi/round8_diagnostics_rule_test.go new file mode 100644 index 000000000..2b9c72d72 --- /dev/null +++ b/internal/httpapi/round8_diagnostics_rule_test.go @@ -0,0 +1,55 @@ +package httpapi + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" +) + +// Issue #1148, round 8 finding 2. +// +// `health.detail` and `diagnostic.cause` are the SAME two strings that +// oauth.RedactServerSecretFields scrubs on GET /api/v1/servers and on the +// /events servers.changed payload, where the rule is the name rule PLUS the +// value-shaped detector. The per-server diagnostics route scrubbed them with +// the name rule ALONE, so a credential under an unrecognised query parameter +// was masked on one door and published on its sibling. + +const round8GHPToken = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + +func TestRedactHealthDetail_UsesTheSharedFreeTextRule(t *testing.T) { + detail := `Post "https://h/mcp?opaque=` + round8GHPToken + `": no such host` + + got := redactHealthDetail(&contracts.HealthStatus{Detail: detail}, false) + hs, ok := got.(*contracts.HealthStatus) + require.True(t, ok) + + assert.NotContains(t, hs.Detail, round8GHPToken, + "the diagnostics door published a credential its sibling REST door masks") + assert.Equal(t, oauth.ScrubUpstreamText(detail), hs.Detail, + "health.detail must be scrubbed with the one shared free-text rule") +} + +func TestRedactDiagnosticCause_UsesTheSharedFreeTextRule(t *testing.T) { + cause := `Get "https://h/mcp?opaque=` + round8GHPToken + `": connection refused` + diag := map[string]interface{}{"cause": cause} + + redactDiagnosticCause(diag, false) + + assert.NotContains(t, diag["cause"], round8GHPToken, + "the diagnostics door published a credential its sibling REST door masks") + assert.Equal(t, oauth.ScrubUpstreamText(cause), diag["cause"], + "diagnostic.cause must be scrubbed with the one shared free-text rule") +} + +func TestRedactHealthDetail_RevealStillOptsOut(t *testing.T) { + detail := "raw " + round8GHPToken + got := redactHealthDetail(&contracts.HealthStatus{Detail: detail}, true) + hs, ok := got.(*contracts.HealthStatus) + require.True(t, ok) + assert.Equal(t, detail, hs.Detail) +} diff --git a/internal/httpapi/round9_config_import_doors_test.go b/internal/httpapi/round9_config_import_doors_test.go new file mode 100644 index 000000000..9d68b25e9 --- /dev/null +++ b/internal/httpapi/round9_config_import_doors_test.go @@ -0,0 +1,335 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" + runtime "github.com/smart-mcp-proxy/mcpproxy-go/internal/runtime" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +// Issue #1148, round 9. Two doors the round-8 inventory never saw. +// +// 1. `POST /api/v1/servers/import{,/json,/path}` echoes the operator's source +// file back with `url`, `command` and `args` RAW, while its CLI twin +// (`upstream import`, buildImportedServersOutput) was redacted in round 8. +// Same data, two doors, one redacted. +// +// 2. `GET /api/v1/config` returns the ENTIRE config — every server's env, +// headers, oauth.client_secret and url credentials, plus the global +// docker_isolation.extra_args — with no rule at all. It is the widest door +// on the tree, and masking it is only half an answer: the raw-JSON editor +// and the onboarding wizard both GET this document and POST it back, so +// without a matching bind-or-refuse on the write path the mask would be +// PERSISTED over the credential — the #1142 corruption this branch has +// spent four rounds preventing. + +const ( + round9Secret = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + round9OAuthSecre = "oauth-client-secret-value" +) + +func round9Config() *config.Config { + return &config.Config{ + APIKey: "super-secret-api-key", + Servers: []*config.ServerConfig{{ + Name: "github", + Protocol: "http", + URL: "https://host/mcp?access_token=" + round9Secret, + Command: "npx", + Args: []string{"--token", round9Secret}, + Env: map[string]string{"GITHUB_TOKEN": round9Secret}, + Headers: map[string]string{"Authorization": "Bearer " + round9Secret}, + OAuth: &config.OAuthConfig{ClientID: "abc", ClientSecret: round9OAuthSecre}, + }}, + DockerIsolation: &config.DockerIsolationConfig{ + Enabled: true, + ExtraArgs: []string{"-e", "API_KEY=" + round9Secret}, + }, + } +} + +// mockRound9ConfigController serves a config with secrets in every shape a +// server can carry one, and captures whatever the write doors resolve. +type mockRound9ConfigController struct { + baseController + live *config.Config + captured *config.Config + applied int +} + +func (m *mockRound9ConfigController) GetCurrentConfig() any { + return &config.Config{APIKey: "test-key"} +} +func (m *mockRound9ConfigController) GetConfig() (*config.Config, error) { return m.live, nil } +func (m *mockRound9ConfigController) GetConfigPath() string { return "/tmp/mcp_config.json" } + +func (m *mockRound9ConfigController) ApplyConfig(cfg *config.Config, _ string) (*runtime.ConfigApplyResult, error) { + clone := *cfg + m.captured = &clone + m.applied++ + return &runtime.ConfigApplyResult{Success: true, AppliedImmediately: true}, nil +} + +func round9Server(t *testing.T, ctrl *mockRound9ConfigController) *Server { + t.Helper() + return NewServer(ctrl, zap.NewNop().Sugar(), nil) +} + +func round9Do(t *testing.T, srv *Server, method, path string, body []byte) *httptest.ResponseRecorder { + t.Helper() + var rdr *bytes.Reader + if body == nil { + rdr = bytes.NewReader(nil) + } else { + rdr = bytes.NewReader(body) + } + req := httptest.NewRequest(method, path, rdr) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-API-Key", "test-key") + w := httptest.NewRecorder() + srv.ServeHTTP(w, req) + return w +} + +// getConfigDocument returns the `config` object GET /api/v1/config published. +func getConfigDocument(t *testing.T, srv *Server) map[string]interface{} { + t.Helper() + w := round9Do(t, srv, http.MethodGet, "/api/v1/config", nil) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + var envelope struct { + Data struct { + Config map[string]interface{} `json:"config"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &envelope)) + require.NotEmpty(t, envelope.Data.Config, "GET /api/v1/config returned no config document") + return envelope.Data.Config +} + +// --- door 2: GET /api/v1/config --------------------------------------------- + +func TestGetConfig_MasksEverySecretItPublishes(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + w := round9Do(t, srv, http.MethodGet, "/api/v1/config", nil) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + body := w.Body.String() + + for _, secret := range []string{round9Secret, round9OAuthSecre, "super-secret-api-key"} { + assert.NotContains(t, body, secret, + "GET /api/v1/config published a credential in the clear. It serves the WHOLE config — "+ + "every server's env, headers, oauth.client_secret and url credentials, plus the global "+ + "docker_isolation.extra_args — and every sibling door masks the same leaves.") + } + + // A mask that eats the document is not a fix: this endpoint backs the + // raw-JSON editor and the Settings form, so every non-secret leaf and every + // key must survive the walk. + document := getConfigDocument(t, srv) + servers, ok := document["mcpServers"].([]interface{}) + require.True(t, ok, "the walk dropped mcpServers") + require.Len(t, servers, 1) + server, ok := servers[0].(map[string]interface{}) + require.True(t, ok) + assert.Equal(t, "github", server["name"], "a server name is not a secret and must stay readable") + assert.Equal(t, "npx", server["command"], "an ordinary command passes the shared leaf rule unchanged") + assert.Contains(t, server, "env", "the env block was dropped rather than masked") + assert.Contains(t, server, "oauth", "the oauth block was dropped rather than masked") + isolation, ok := document["docker_isolation"].(map[string]interface{}) + require.True(t, ok, "the walk dropped the global docker_isolation block") + assert.Equal(t, true, isolation["enabled"], "a non-secret isolation flag must survive the walk") +} + +// The mask is only safe if the write path can put it back. Both the raw-JSON +// editor and the onboarding wizard GET this document, splice one field, and +// POST the whole thing to /config/apply. +func TestConfigApply_RoundTripOfTheMaskedReadKeepsTheCredentials(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + document := getConfigDocument(t, srv) + document["quarantine_enabled"] = false // the one field the operator changed + + body, err := json.Marshal(document) + require.NoError(t, err) + w := round9Do(t, srv, http.MethodPost, "/api/v1/config/apply", body) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + + require.NotNil(t, ctrl.captured, "ApplyConfig was never called") + require.Len(t, ctrl.captured.Servers, 1) + got := ctrl.captured.Servers[0] + + assert.Equal(t, round9Secret, got.Env["GITHUB_TOKEN"], "the env mask was persisted over the credential") + assert.Equal(t, "Bearer "+round9Secret, got.Headers["Authorization"], "the header mask was persisted over the credential") + assert.Equal(t, "https://host/mcp?access_token="+round9Secret, got.URL, "the url mask was persisted over the credential") + require.NotNil(t, got.OAuth) + assert.Equal(t, round9OAuthSecre, got.OAuth.ClientSecret, "the oauth mask was persisted over the credential") + assert.Equal(t, "super-secret-api-key", ctrl.captured.APIKey, "the api_key mask was persisted over the key") + assert.Equal(t, []string{"--token", round9Secret}, got.Args, + "the argv mask was persisted over the credential. argv has no key to bind to leaf by leaf, so the "+ + "only binding it can ever have is the one this round trip gives it: the server block came back "+ + "byte for byte as the read door rendered it, which is proof the caller changed nothing in it.") + assert.Equal(t, []string{"-e", "API_KEY=" + round9Secret}, ctrl.captured.DockerIsolation.ExtraArgs, + "the global docker_isolation.extra_args mask was persisted over the credential") +} + +// The binding above is proof of NO CHANGE, not a licence to revert. Edit +// anything inside the block and it is gone — and an argv slot has nothing else +// to bind to, so the write must be refused rather than written through. +func TestConfigApply_RefusesAnArgvMaskUnderAnEditedServer(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + document := getConfigDocument(t, srv) + servers, ok := document["mcpServers"].([]interface{}) + require.True(t, ok) + require.NotEmpty(t, servers) + server, ok := servers[0].(map[string]interface{}) + require.True(t, ok) + server["protocol"] = "sse" // an ordinary edit, but it breaks the whole-block echo + + body, err := json.Marshal(document) + require.NoError(t, err) + w := round9Do(t, srv, http.MethodPost, "/api/v1/config/apply", body) + + assert.Equal(t, http.StatusBadRequest, w.Code, + "an argv mask under an EDITED server was accepted; nothing binds an argv slot but its index") + assert.Contains(t, w.Body.String(), "args", + "the refusal must name the field the operator has to resend") + assert.Nil(t, ctrl.captured, "ApplyConfig must not run on a refused write") +} + +// And the other half: a mask the write path cannot BIND must be refused, not +// written through. `args` has only an index to bind to, which is no binding — +// the decision table has said `refuse` for it since round 6. +func TestConfigApply_RefusesAMaskItCannotBind(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + document := getConfigDocument(t, srv) + servers, ok := document["mcpServers"].([]interface{}) + require.True(t, ok, "config document carries no mcpServers array") + require.NotEmpty(t, servers) + server, ok := servers[0].(map[string]interface{}) + require.True(t, ok) + // The operator renamed the server: nothing binds the masked argv token back. + server["name"] = "github-renamed" + + body, err := json.Marshal(document) + require.NoError(t, err) + w := round9Do(t, srv, http.MethodPost, "/api/v1/config/apply", body) + + assert.Equal(t, http.StatusBadRequest, w.Code, + "an unbindable mask was ACCEPTED on /config/apply and would be persisted over the credential") + assert.Nil(t, ctrl.captured, "ApplyConfig must not run on a refused write") +} + +func TestPatchConfig_RevertsWhatItCanBindAndRefusesWhatItCannot(t *testing.T) { + t.Run("bound by key, reverted", func(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + masked := oauth.LiveRedaction.EnvValue("GITHUB_TOKEN", round9Secret) + require.NotEqual(t, round9Secret, masked, "precondition: the read door masks this env value") + patch := map[string]interface{}{ + "mcpServers": []interface{}{map[string]interface{}{ + "name": "github", + "env": map[string]interface{}{"GITHUB_TOKEN": masked}, + }}, + } + body, err := json.Marshal(patch) + require.NoError(t, err) + w := round9Do(t, srv, http.MethodPatch, "/api/v1/config", body) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + + require.NotNil(t, ctrl.captured) + require.Len(t, ctrl.captured.Servers, 1) + assert.Equal(t, round9Secret, ctrl.captured.Servers[0].Env["GITHUB_TOKEN"], + "an echoed env mask bound to its own key must be reverted, never persisted") + }) + + t.Run("no binding, refused", func(t *testing.T) { + ctrl := &mockRound9ConfigController{live: round9Config()} + srv := round9Server(t, ctrl) + + maskedArgs := oauth.LiveRedaction.Argv([]string{"--token", round9Secret}) + require.NotEqual(t, round9Secret, maskedArgs[1], "precondition: the read door masks this argv token") + patch := map[string]interface{}{ + "mcpServers": []interface{}{map[string]interface{}{ + "name": "github-renamed", + "args": []interface{}{maskedArgs[0], maskedArgs[1]}, + }}, + } + body, err := json.Marshal(patch) + require.NoError(t, err) + w := round9Do(t, srv, http.MethodPatch, "/api/v1/config", body) + + assert.Equal(t, http.StatusBadRequest, w.Code, + "an argv mask with nothing to bind to was ACCEPTED and would be persisted over the credential") + assert.Nil(t, ctrl.captured, "ApplyConfig must not run on a refused write") + }) +} + +// --- door 1: the import preview --------------------------------------------- + +type mockRound9ImportController struct { + baseController +} + +func (m *mockRound9ImportController) GetCurrentConfig() any { + return &config.Config{APIKey: "test-key"} +} +func (m *mockRound9ImportController) AddServer(_ context.Context, _ *config.ServerConfig) error { + return nil +} + +func TestImportPreview_MasksTheSameLeavesItsCLITwinDoes(t *testing.T) { + srv := NewServer(&mockRound9ImportController{}, zap.NewNop().Sugar(), nil) + + source := `{"mcpServers":{"github":{"command":"npx","args":["--token","` + round9Secret + `"],` + + `"env":{"GITHUB_TOKEN":"` + round9Secret + `"}},` + + `"remote":{"url":"https://host/mcp?access_token=` + round9Secret + `"}}}` + body, err := json.Marshal(ImportRequest{Content: source}) + require.NoError(t, err) + + req := httptest.NewRequest(http.MethodPost, "/api/v1/servers/import/json?preview=true", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-API-Key", "test-key") + w := httptest.NewRecorder() + srv.ServeHTTP(w, req) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + + assert.NotContains(t, w.Body.String(), round9Secret, + "POST /api/v1/servers/import echoed the operator's source file back with url/command/args RAW. "+ + "Its CLI twin (upstream import → buildImportedServersOutput) masks exactly these leaves with the "+ + "shared LIVE rule; same data, two doors, one answer.") + + // And the readable half of the rule still holds: an ordinary command and an + // ordinary flag pass through byte-identical. + var wrapped wrappedImportResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &wrapped)) + require.Len(t, wrapped.Data.Imported, 2) + for _, imported := range wrapped.Data.Imported { + if imported.Command != "" { + assert.Equal(t, "npx", imported.Command, "an ordinary command must survive the rule unchanged") + require.NotEmpty(t, imported.Args) + assert.Equal(t, "--token", imported.Args[0], "an ordinary flag must survive the rule unchanged") + } + if imported.URL != "" { + assert.True(t, strings.HasPrefix(imported.URL, "https://host/mcp"), + "the URL is masked component-wise, so scheme and host stay readable: %q", imported.URL) + } + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 80500c9d6..abba5081c 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -4260,7 +4260,8 @@ func (s *Server) handleGetDiagnostics(w http.ResponseWriter, r *http.Request) { if server.LastError != "" { errMsg := server.LastError if !reveal { - errMsg = oauth.RedactSensitiveData(errMsg) + // Round 8 finding 2: the ONE shared free-text rule. + errMsg = oauth.ScrubUpstreamText(errMsg) } upstreamErrors = append(upstreamErrors, contracts.DiagnosticIssue{ Type: "error", @@ -4753,15 +4754,52 @@ func (s *Server) handleGetConfig(w http.ResponseWriter, r *http.Request) { return } + // Issue #1148, round 9: this endpoint used to serve the RAW config.Config — + // every server's env, headers, oauth.client_secret and url credentials, the + // global docker_isolation.extra_args and the api_key — while every sibling + // door masked the same leaves. It was the widest door on the tree and it + // had no row in the decision table at all. + // + // The shared LIVE walk answers here too. Its write twin is + // oauth.UnmaskLiveConfigTree, called by handleApplyConfig and + // handlePatchConfig below: the raw-JSON editor and the onboarding wizard + // GET this document and POST it straight back, so masking the read without + // the matching bind-or-refuse would persist the mask over the credential — + // the #1142 corruption. The two are one change. + // + // `reveal_secret_headers` opts out exactly as it does on GET + // /api/v1/servers and the /events payload; the flag is read off the RUNNING + // config, not the desired one, so a client cannot widen its own read by + // staging the flag and re-reading before the restart. + published := cfg + if !s.revealSecretHeaders() { + published = oauth.RedactedConfig(cfg) + if published == nil { + // Fail CLOSED. Serving the unmasked config because the walk could + // not round-trip is the fail-open shape this issue is made of. + s.logger.Errorw("Failed to redact configuration for publication") + s.writeError(w, r, http.StatusInternalServerError, "Failed to get configuration") + return + } + } + // Convert config to contracts type for consistent API response response := contracts.GetConfigResponse{ - Config: contracts.ConvertConfigToContract(cfg), + Config: contracts.ConvertConfigToContract(published), ConfigPath: s.controller.GetConfigPath(), } s.writeSuccess(w, response) } +// revealSecretHeaders reports the operator's opt-out of secret masking. Read +// from the RUNNING config, which is the same source redactServerSecrets uses, +// so the two doors cannot disagree about whether this response is masked. +func (s *Server) revealSecretHeaders() bool { + cfg, err := s.controller.GetConfig() + return err == nil && cfg != nil && cfg.RevealSecretHeaders +} + // handleValidateConfig godoc // @Summary Validate configuration // @Description Validates a provided MCPProxy configuration without applying it. Checks for syntax errors, invalid server definitions, conflicting settings, and other configuration issues. @@ -4825,12 +4863,36 @@ func (s *Server) handleApplyConfig(w http.ResponseWriter, r *http.Request) { return } - var cfg config.Config - if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { + // Decoded as a generic document first, so the masks a client echoed back + // from GET /api/v1/config can be resolved BEFORE anything is typed or + // persisted (issue #1148, round 9). UseNumber keeps integers exact through + // the round trip instead of degrading them to float64. + decoder := json.NewDecoder(r.Body) + decoder.UseNumber() + var document map[string]interface{} + if err := decoder.Decode(&document); err != nil { s.writeError(w, r, http.StatusBadRequest, "Invalid JSON payload") return } + stored, err := s.desiredConfigForPatch() + if err != nil { + s.logger.Errorw("Failed to read configuration for apply", "error", err) + s.writeError(w, r, http.StatusInternalServerError, "Failed to read configuration") + return + } + + // Revert what binds to a key, refuse what does not. Without this the + // raw-JSON editor's own GET → edit → POST round trip would write the read + // door's masks over the operator's credentials. + resolved, err := oauth.UnmaskLiveConfigDocument(document, stored) + if err != nil { + s.logger.Warnw("Refused a configuration write carrying an unbindable mask", "error", err) + s.writeError(w, r, http.StatusBadRequest, err.Error()) + return + } + cfg := *resolved + // Get config path from controller cfgPath := s.controller.GetConfigPath() @@ -4989,6 +5051,22 @@ func (s *Server) handlePatchConfig(w http.ResponseWriter, r *http.Request) { return } + // Issue #1148, round 9: resolve any mask the client echoed back BEFORE the + // merge. A patch is exactly the shape that used to corrupt — the client + // read `env: {GITHUB_TOKEN: "••••56 (40 chars)"}` off a read door and sent + // it back — and the merge would have written the mask straight over the + // stored credential. Bound by key it is reverted; unbound (an argv slot, a + // renamed server) the write is refused. + resolvedPatch, err := oauth.UnmaskLiveConfigTree(patchMap, cfg) + if err != nil { + s.logger.Warnw("Refused a configuration patch carrying an unbindable mask", "error", err) + s.writeError(w, r, http.StatusBadRequest, err.Error()) + return + } + if m, ok := resolvedPatch.(map[string]interface{}); ok { + patchMap = m + } + deepMergeJSON(baseMap, patchMap) mergedBytes, err := json.Marshal(baseMap) @@ -5421,13 +5499,20 @@ func (s *Server) handleAddFromRegistry(w http.ResponseWriter, r *http.Request) { return } + // Issue #1148, round 8: the MCP twin of this handler + // (`upstream_servers add_from_registry`) has sourced this echo from the + // shared redacted view since round 1; this one still read the struct, so a + // registry entry carrying `?token=…` or a `--api-key …` arg vector was + // masked on one surface and republished on the other. Same summary, same + // view, one answer. + registryView := oauth.RedactedConfigView("", cfg) s.writeSuccess(w, contracts.AddFromRegistryData{ Server: contracts.AddedServerSummary{ Name: cfg.Name, Protocol: cfg.Protocol, - Command: cfg.Command, - Args: cfg.Args, - URL: cfg.URL, + Command: viewString(registryView, "command", cfg.Command), + Args: oauth.LiveRedaction.Argv(cfg.Args), + URL: viewString(registryView, "url", cfg.URL), Enabled: cfg.Enabled, Quarantined: cfg.Quarantined, }, @@ -5472,15 +5557,7 @@ func (s *Server) handleAddRegistrySource(w http.ResponseWriter, r *http.Request) } s.writeSuccess(w, contracts.AddRegistrySourceData{ - Registry: contracts.RegistrySummary{ - ID: entry.ID, - Name: entry.Name, - URL: entry.URL, - ServersURL: entry.ServersURL, - Protocol: entry.Protocol, - Provenance: entry.Provenance, - Trusted: entry.IsTrusted(), - }, + Registry: redactedRegistrySummary(entry), }) } @@ -5517,15 +5594,7 @@ func (s *Server) handleRemoveRegistrySource(w http.ResponseWriter, r *http.Reque } s.writeSuccess(w, contracts.RemoveRegistrySourceData{ - Registry: contracts.RegistrySummary{ - ID: entry.ID, - Name: entry.Name, - URL: entry.URL, - ServersURL: entry.ServersURL, - Protocol: entry.Protocol, - Provenance: entry.Provenance, - Trusted: entry.IsTrusted(), - }, + Registry: redactedRegistrySummary(entry), }) } @@ -5570,15 +5639,7 @@ func (s *Server) handleEditRegistrySource(w http.ResponseWriter, r *http.Request } s.writeSuccess(w, contracts.EditRegistrySourceData{ - Registry: contracts.RegistrySummary{ - ID: entry.ID, - Name: entry.Name, - URL: entry.URL, - ServersURL: entry.ServersURL, - Protocol: entry.Protocol, - Provenance: entry.Provenance, - Trusted: entry.IsTrusted(), - }, + Registry: redactedRegistrySummary(entry), }) } @@ -5817,13 +5878,15 @@ func (s *Server) handleGetDockerStatus(w http.ResponseWriter, r *http.Request) { } response := map[string]interface{}{ - "docker_available": dockerAvailable, - "isolation_enabled": isolationEnabled, - "recovery_mode": status.RecoveryMode, - "failure_count": status.FailureCount, - "attempts_since_up": status.AttemptsSinceUp, - "last_attempt": status.LastAttempt, - "last_error": status.LastError, + "docker_available": dockerAvailable, + "isolation_enabled": isolationEnabled, + "recovery_mode": status.RecoveryMode, + "failure_count": status.FailureCount, + "attempts_since_up": status.AttemptsSinceUp, + "last_attempt": status.LastAttempt, + // Round 8: a Docker daemon failure quotes the `docker run` command line + // it choked on, `-e API_KEY=…` and all. One shared free-text rule. + "last_error": oauth.ScrubUpstreamText(status.LastError), "last_successful_at": status.LastSuccessfulAt, } diff --git a/internal/management/diagnostics.go b/internal/management/diagnostics.go index 17d465ddc..2988db17a 100644 --- a/internal/management/diagnostics.go +++ b/internal/management/diagnostics.go @@ -71,7 +71,10 @@ func (s *service) Doctor(ctx context.Context) (*contracts.Diagnostics, error) { if s.config != nil && s.config.RevealSecretHeaders { return msg } - return oauth.RedactSensitiveData(msg) + // Round 8 finding 2: the ONE shared free-text rule — the same one + // oauth.RedactServerSecretFields applies to health.detail and last_error + // on the REST/SSE door these strings are copied from. + return oauth.ScrubUpstreamText(msg) } // Aggregate diagnostics from Health.Action (single source of truth) diff --git a/internal/management/round8_diagnostics_rule_test.go b/internal/management/round8_diagnostics_rule_test.go new file mode 100644 index 000000000..0953e771e --- /dev/null +++ b/internal/management/round8_diagnostics_rule_test.go @@ -0,0 +1,38 @@ +package management + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" +) + +// Issue #1148, round 8 finding 2. The doctor aggregator copies `health.detail` +// and `last_error` into UpstreamErrors.ErrorMessage — the same two strings +// oauth.RedactServerSecretFields scrubs with the name rule PLUS the +// value-shaped detector on the REST/SSE door. It ran the name rule alone. +func TestDoctor_UsesTheSharedFreeTextRuleForUpstreamErrors(t *testing.T) { + const token = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + msg := `Post "https://h/mcp?opaque=` + token + `": no such host` + + logger := zaptest.NewLogger(t).Sugar() + rt := newMockRuntime() + rt.servers = []map[string]interface{}{ + {"name": "leaky", "last_error": msg}, + } + + svc := NewService(rt, &config.Config{}, "", &mockEventEmitter{}, nil, logger) + diag, err := svc.Doctor(context.Background()) + require.NoError(t, err) + require.Len(t, diag.UpstreamErrors, 1) + + assert.NotContains(t, diag.UpstreamErrors[0].ErrorMessage, token, + "the doctor door published a credential its sibling REST door masks") + assert.Equal(t, oauth.ScrubUpstreamText(msg), diag.UpstreamErrors[0].ErrorMessage, + "UpstreamErrors.ErrorMessage must use the one shared free-text rule") +} diff --git a/internal/oauth/configview.go b/internal/oauth/configview.go new file mode 100644 index 000000000..ba4f73e48 --- /dev/null +++ b/internal/oauth/configview.go @@ -0,0 +1,217 @@ +package oauth + +import ( + "encoding/json" + "reflect" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" +) + +// Issue #1148, round 9. The WHOLE-CONFIG door and its write twin. +// +// `GET /api/v1/config` serves the entire config.Config: every server's env, +// headers, oauth.client_secret and url credentials, the global +// docker_isolation.extra_args, and the api_key. It is the widest read door on +// the tree and it had no rule at all — the round-8 inventory could not see it, +// because the leak's shape is "a whole config handed to a publish sink", not +// "a leaf copied into a map". +// +// Masking it is only half an answer. The raw-JSON editor and the onboarding +// wizard both GET this document, change one field, and POST the whole thing +// back to `/api/v1/config/apply`; the macOS tray and the Settings page PATCH +// it. Masking the read without a matching revert on the write would persist +// the MASK over the credential — the #1142 corruption this branch has spent +// four rounds preventing. So the two functions below are written as one pair +// and neither is useful alone. + +// RedactedConfig returns a masked deep copy of a whole config, using the same +// generic LIVE walk every other door masks a nested server block with. +// +// Walking rather than enumerating is the durable half, for exactly the reason +// round 7 gave: a field added to config.Config later is masked because the walk +// reaches it, instead of being published in the clear until somebody remembers. +// +// Returns nil when the walk cannot round-trip. A caller must treat that as a +// failure and serve an error: publishing the unmasked config instead is the +// fail-open shape this whole issue is made of. +func RedactedConfig(cfg *config.Config) *config.Config { + if cfg == nil { + return nil + } + masked := *cfg + if !LiveRedaction.RedactNested("", &masked) { + return nil + } + return &masked +} + +// UnmaskLiveConfigTree is the write twin of RedactedConfig: it reverts every +// mask a live read door rendered that can be BOUND TO A KEY, and refuses +// whatever is left. +// +// `incoming` is what the client sent — a whole config document on +// `/config/apply`, a partial one on `PATCH /config` — and `stored` is what the +// proxy holds. Both are walked as generic JSON, so a PARTIAL tree works: the +// bindings are per key at each level, not positional. +// +// The bindings are exactly the ones ServerFieldMaskDecisions already records, +// applied one level up: +// +// - a MAP key (`env`, `headers`, `oauth.extra_params`) and a named struct +// field (`url`, `command`, `api_key`, `oauth.client_secret`) bind, because +// the caller cannot restate the key without also restating what the secret +// is for. +// - the `mcpServers` array binds BY NAME, never by index: a caller who +// renames a server has not told the proxy which stored secret they meant. +// - an argv / scopes / extra_args slot binds to NOTHING — an index is not a +// binding — so its mask survives the walk and CheckServerWriteMasks refuses +// the write, which is the `refuse` decision the table has recorded since +// round 6. +// - and one binding wider than all of them: a SUBTREE echoed back byte for +// byte as the read door rendered it is proof the caller changed nothing in +// it, so the stored subtree is restored wholesale. That is the same +// binding UnmaskLiveURL has used since round 4 for a whole URL, one level +// up — and it is what keeps the raw-JSON editor and the onboarding wizard +// working: they GET the document, change one top-level field and POST it +// back, so every server block is byte-identical and none of them has to +// bind leaf by leaf. Change anything inside a block and the equality +// fails, the walk narrows to per-key bindings, and an argv mask under an +// edited server is refused exactly as it should be. +// +// Anything the walk did not revert and still carries a mask this proxy rendered +// fails the write CLOSED, including a field added to config.Config later. +func UnmaskLiveConfigTree(incoming, stored interface{}) (interface{}, error) { + reverted := unmaskConfigNode("", NormalizeForRedaction(incoming), NormalizeForRedaction(stored)) + if err := CheckServerWriteMasks("config", reverted); err != nil { + return nil, err + } + return reverted, nil +} + +// unmaskConfigNode mirrors Redaction.Value one node at a time: whatever the +// read walk rendered for a leaf under this key is what the write walk compares +// against. The two must stay in step, which is why both are keyed the same way +// and neither carries its own field list. +func unmaskConfigNode(key string, incoming, stored interface{}) interface{} { + // The widest binding first: a subtree the caller echoed back exactly as the + // read door rendered it is unchanged, so the stored one goes back verbatim. + // Equality with mask(stored) is proof — the same proof UnmaskLiveURL takes + // for a whole URL, one level up. + // + // It is NOT offered to an argv vector on its own. `key` there is `args` / + // `extra_args`, and a vector standing alone under an edited block is + // exactly what CheckArgvMaskEcho refuses at the server write doors: the + // caller supplies the vector and `command` together, so an echoed token + // proves nothing about the slot it came from. Reached as part of a whole + // BLOCK echo it is restored, because there the caller restated the entire + // server — name, command, every argument — byte for byte, which is a + // strictly stronger binding than the one that check rejects. The two doors + // need that difference: `PATCH /api/v1/servers/{id}` lets a caller OMIT + // `args` to leave it alone, and a whole-document `POST /config/apply` has + // no such escape — omitting the field there deletes it. + if stored != nil && incoming != nil && !IsArgvFieldName(key) { + if reflect.DeepEqual(incoming, LiveRedaction.Value(key, stored)) { + return stored + } + } + switch in := incoming.(type) { + case map[string]interface{}: + storedMap, _ := stored.(map[string]interface{}) + out := make(map[string]interface{}, len(in)) + isHeaders := IsHeadersFieldName(key) + isEnv := IsEnvFieldName(key) + for k, v := range in { + var sv interface{} + if storedMap != nil { + sv = storedMap[k] + } + if s, ok := v.(string); ok { + out[k] = revertLeaf(k, s, sv, isHeaders, isEnv) + continue + } + out[k] = unmaskConfigNode(k, v, sv) + } + return out + case []interface{}: + // An argv vector offers only an INDEX to bind to, which is no binding: + // the caller supplies the whole vector AND `command` in one request, so + // a restored token could be moved to a slot it was never read from. + // Leave it; the residual net refuses it. + if IsArgvFieldName(key) { + return in + } + byName := map[string]interface{}{} + if storedSlice, ok := stored.([]interface{}); ok { + for _, el := range storedSlice { + m, ok := el.(map[string]interface{}) + if !ok { + continue + } + if name, ok := m["name"].(string); ok && name != "" { + byName[name] = m + } + } + } + out := make([]interface{}, len(in)) + for i, el := range in { + m, ok := el.(map[string]interface{}) + if !ok { + // A plain string element (oauth.scopes, docker_isolation. + // extra_args): nothing but its index identifies it. Refuse. + out[i] = el + continue + } + name, _ := m["name"].(string) + out[i] = unmaskConfigNode(key, m, byName[name]) + } + return out + default: + return incoming + } +} + +// revertLeaf restores one string leaf echoed back exactly as the live read walk +// rendered it, bound to its own key. +func revertLeaf(key, incoming string, stored interface{}, isHeaders, isEnv bool) string { + storedStr, ok := stored.(string) + if !ok || storedStr == "" || incoming == "" { + return incoming + } + var rendered string + switch { + case isHeaders: + rendered = LiveRedaction.HeaderValue(key, storedStr) + case isEnv: + rendered = LiveRedaction.EnvValue(key, storedStr) + default: + rendered = LiveRedaction.Leaf(key, storedStr) + } + if incoming == rendered { + return storedStr + } + return incoming +} + +// UnmaskLiveConfigDocument is UnmaskLiveConfigTree for a whole config document +// a client PUT/POSTed: it resolves the masks against `stored` and decodes the +// result into a fresh config.Config. +// +// Decoding into a FRESH value rather than over an existing one is the same rule +// RedactNested follows: encoding/json reuses an existing slice's backing array, +// so decoding over a value that still aliases the stored config would overwrite +// live credentials with their own masks. +func UnmaskLiveConfigDocument(incoming map[string]interface{}, stored *config.Config) (*config.Config, error) { + resolved, err := UnmaskLiveConfigTree(incoming, stored) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(resolved) + if err != nil { + return nil, err + } + fresh := &config.Config{} + if err := json.Unmarshal(encoded, fresh); err != nil { + return nil, err + } + return fresh, nil +} diff --git a/internal/oauth/redactview.go b/internal/oauth/redactview.go index 94c37cf13..e5be54a77 100644 --- a/internal/oauth/redactview.go +++ b/internal/oauth/redactview.go @@ -334,6 +334,86 @@ func MaskDetectedSecrets(s string) string { return masked } +// ScrubUpstreamText is the ONE rule for free-form text that originated OUTSIDE +// mcpproxy's own structured fields — an upstream connection error, a health +// detail, a diagnostic cause, a line tailed from a child server's log. +// +// Such a string has no enclosing key to judge it by, so BOTH rules run: the +// name rule (RedactSensitiveData, which masks `=` shapes, bearer +// tokens and URL userinfo) and then the value-shaped detector (which masks a +// vendor-formatted credential wherever it sits — `?opaque=ghp_…`, a token a +// child process printed). +// +// Issue #1148, round 8 findings 1 and 2: FIVE doors published the same three +// strings (`last_error`, `health.detail`, `diagnostic.cause`) and disagreed +// about the rule. `RedactServerSecretFields` (REST + SSE) and the MCP view ran +// both rules; `upstream.Manager.GetStats` (`upstream_stats` on /api/v1/status +// and the SSE status event), the per-server diagnostics route, the doctor route +// and the management diagnostics aggregator ran only the name rule — so the +// same error string was masked on one door of one response and published in the +// clear on another. Same strings, one rule, one answer. +func ScrubUpstreamText(s string) string { + if s == "" { + return s + } + return MaskDetectedSecrets(RedactSensitiveData(s)) +} + +// RedactUpstreamStats masks the secret-bearing leaves of an `upstream_stats` +// map IN PLACE and returns it. +// +// `upstream_stats` is served verbatim on GET /api/v1/status, embedded in the +// /api/v1/servers response and pushed on every SSE `status` event, so its +// per-server entries sit behind the same trust boundary as the server list on +// those same payloads. +// +// Issue #1148, rounds 4 and 8. The map is built by TWO implementations — +// upstream.Manager.GetStats (the fallback) and server.Server.GetUpstreamStats +// (the StateView path that actually runs) — and they disagreed: round 4 masked +// the first with name-only rules and never touched the second at all, so the +// path that serves nearly every request published `url` and `last_error` +// verbatim. One rule, applied at the wire boundary both implementations pass +// through, is the only shape that cannot drift again. +// +// Returns the map unchanged when `reveal` is true, honouring the same +// reveal_secret_headers opt-out every sibling read door honours. +func RedactUpstreamStats(stats map[string]interface{}, reveal bool) map[string]interface{} { + if stats == nil || reveal { + return stats + } + servers, ok := stats["servers"].(map[string]interface{}) + if !ok { + return stats + } + for _, raw := range servers { + entry, ok := raw.(map[string]interface{}) + if !ok { + continue + } + RedactUpstreamStatsEntry(entry) + } + return stats +} + +// RedactUpstreamStatsEntry masks one per-server `upstream_stats` entry in place. +// +// `url` takes the shared LIVE URL rule (the name rule PLUS the value-shaped +// detector) and `last_error` the shared free-form rule, which are the same two +// rules the server list in the SAME response applies to the same two strings. +// Every other leaf of the entry — state, counters, the upstream's self-reported +// name and version — is proxy-computed and carries no operator value. +func RedactUpstreamStatsEntry(entry map[string]interface{}) { + if entry == nil { + return + } + if u, ok := entry["url"].(string); ok && u != "" { + entry["url"] = LiveRedaction.URLValue(u) + } + if e, ok := entry["last_error"].(string); ok && e != "" { + entry["last_error"] = ScrubUpstreamText(e) + } +} + // maskRendering is one function that RENDERS a secret as a mask on a read door, // paired with the MARKER constant its output is built from and a probe input it // is guaranteed to mask. @@ -370,6 +450,24 @@ var maskRenderings = []maskRendering{ {"oauth.RedactSensitiveData/userinfo", redactedMarker, "postgres://u:supersecretvalue@host/db", RedactSensitiveData}, {"oauth.RedactURL", redactedMarker, "https://host/mcp?access_token=supersecretvalue", RedactURL}, {"security.MaskValue/prefix", securityMaskSuffixMarker(), "AKIAIOSFODNN7EXAMPLE", MaskDetectedSecrets}, + // Round 8. A URL is re-serialised through net/url after its components are + // masked, so the mask reaches the wire PERCENT-ENCODED + // (`%E2%80%A2%E2%80%A2%E2%80%A2%E2%80%A2ue+%2816+chars%29`) — a rendering + // no marker in the set matched. Every live read door publishes it (`url`, + // a connection string in `env`, `oauth.extra_params`), and + // UnmaskLiveURL's residual check is ContainsMaskMarker — so an echoed + // masked URL that the key-bound revert could not bind was ACCEPTED and + // persisted over the credential, which is the #1142 corruption this branch + // exists to prevent. Found by TestExportedStringFuncs_RenderOnlyRecognisedMasks. + {"oauth.LiveRedaction.URLValue/query", escapedMarker(bulletMarker), "https://host/mcp?access_token=supersecretvalue", LiveRedaction.URLValue}, + {"oauth.LiveRedaction.URLValue/userinfo", escapedMarker(bulletMarker), "postgres://u:supersecretvalue@host/db", LiveRedaction.URLValue}, +} + +// escapedMarker is a marker as it appears once net/url has re-serialised the +// URL that carries it. Derived with the same escaper net/url uses, so a change +// there cannot silently narrow the net. +func escapedMarker(marker string) string { + return url.QueryEscape(marker) } // maskOAuthSecret / maskExtraParams are deliberately NOT in the registry. They @@ -410,12 +508,19 @@ func deriveMaskMarkers() []string { } for _, r := range maskRenderings { add(r.marker) + // A marker that travels inside a URL reaches the wire percent-encoded, + // and the fail-closed net has to recognise BOTH spellings: the read + // doors publish the escaped one and the write doors compare against + // whatever the client echoes back. Deriving the escaped form here + // rather than listing it keeps the two in step by construction. + add(escapedMarker(r.marker)) } // security.MaskValue has a SECOND rendering for values too short to keep a // prefix; taking the package's whole marker set rather than only the one // the registry probe exercises is what keeps that from being a hole. for _, marker := range security.MaskMarkers() { add(marker) + add(escapedMarker(marker)) } return out } diff --git a/internal/oauth/round8_doorscan_test.go b/internal/oauth/round8_doorscan_test.go new file mode 100644 index 000000000..b6bb2b0f6 --- /dev/null +++ b/internal/oauth/round8_doorscan_test.go @@ -0,0 +1,1224 @@ +package oauth + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Issue #1148, round 8 — the structural check the previous seven rounds needed. +// +// Every round of this review found the same shape: a rule applied at one door +// and not at its sibling. Rounds 5, 6, 7 and 8 each fixed the doors that were +// reported and left the next one open, because nothing enumerated the doors. A +// DOOR is any function that puts a server-derived string onto a wire a client +// can read — an MCP tool response, a REST handler, an SSE payload, CLI JSON, +// diagnostics, status/stats, the tray adapter, the server edition. +// +// The scan below finds them from the SOURCE rather than from memory: every +// function in the door packages that puts a secret-bearing leaf of a server +// config / contract (or a raw config struct) into a response map, a +// contracts.* literal, or a publish sink. Each one must appear in +// serverPayloadDoors with the rule it uses — or with the reason it needs none. +// +// A new door therefore cannot be added silently: the scan finds it, the +// inventory does not have it, and this test goes red in the same edit. A door +// that is deleted or renamed fails too, so the inventory cannot rot. + +// rawServerLeafDoors is the exemption table for the RAW-LEAF scan: every +// function that still puts an unredacted server leaf into a payload shape, and +// the reason that is correct there. A function NOT in this table that the scan +// finds fails the test — that is the check that would have caught rounds 5, 6, +// 7 and 8 in one go. +// +// Nothing here may be a client-facing wire carrying an unredacted credential. +// Each row is either an INTERNAL projection whose only route to a client runs +// through a named shared-rule door, a value that is not operator-supplied, or a +// payload redacted at the return statement of the same function. +var rawServerLeafDoors = map[string]string{ + // --- internal projections; redacted at the serialization boundary --- + "internal/runtime/runtime.go:(m).GetAllServers": "the generic server map every REST/SSE consumer reads. " + + "Masked by oauth.RedactServerSecretFields at httpapi.redactServerSecrets (GET /api/v1/servers and its " + + "single-server children) and at runtime's /events servers.changed publisher; the MCP door builds its own " + + "payload from redactedServerView instead. Redacting here as well would double-mask and break the " + + "reveal_secret_headers opt-out, which is resolved at those boundaries.", + "internal/runtime/runtime.go:(m).getAllServersLegacy": "storage fallback of GetAllServers; same boundary, same consumers.", + "internal/server/server.go:(m).GetAllServers": "the tray/legacy projection behind the same httpapi + SSE boundary.", + "internal/server/server.go:(m).getAllServersLegacy": "storage fallback of the above; same boundary.", + "internal/runtime/isolation_projection.go:buildIsolationMaps": "builds the isolation sub-map of GetAllServers; masked by the shared walk in RedactServerSecretFields and by isolationValues on the MCP door.", + "internal/management/service.go:(m).ListServers": "builds contracts.Server from the runtime projection. Redacted by " + + "httpapi.redactServerSecrets (personal edition) and by listAdminServers (server edition); both call " + + "oauth.RedactServerSecretFields.", + + // --- redacted at the return of the same function --- + "internal/server/server.go:(m).GetUpstreamStats": "the per-server entries are built raw and the whole map is " + + "handed to oauth.RedactUpstreamStats at every return, reveal-gated. Issue #1148 round 8: this is the " + + "upstream_stats implementation that actually runs (the StateView path); round 4 masked only the " + + "upstream.Manager fallback.", + + // --- already-redacted input; the function holds no config --- + "cmd/mcpproxy-tray/internal/api/adapter.go:(m).GetAllServers": "re-shapes the payload the tray fetched from " + + "GET /api/v1/servers, which the core redacted before it left the process. The tray has no config and no " + + "second source for these fields.", + "cmd/mcpproxy-tray/internal/api/adapter.go:(m).GetQuarantinedServers": "same already-redacted REST payload.", + "internal/httpapi/activity.go:storageToContractActivity": "activity rows are scrubbed at WRITE " + + "(server.scrubUpstreamTextForAudit / the audit-policy walk) and masked again on read by " + + "Server.maskActivityPayloads before this projection is served.", + "internal/httpapi/activity.go:storageToContractActivityForExport": "same rows, same two passes; the export route adds bodies, not new raw config leaves.", + + // --- not operator-supplied --- + "cmd/mcpproxy/upstream_cmd.go:runUpstreamListFromConfig": "health.detail here is prose generated by " + + "health.CalculateHealth from a fixed disconnected input; no upstream text and no config string reaches it.", + "cmd/mcpproxy/registry_cmd.go:newRegistrySearchCmd": "registry SEARCH results are catalogue entries fetched from a remote registry, not operator configuration.", + "internal/runtime/runtime.go:(m).SearchRegistryServers": "same remote catalogue entries; the urls are the catalogue's.", + "internal/upstream/core/connection_oauth.go:(m).emptyClientIDFlowError": "the url in this message is the " + + "DISCOVERED authorization-server endpoint, not the configured upstream url.", + "internal/upstream/core/connection_oauth.go:(m).handleOAuthAuthorization": "discovered authorization/token endpoints in zap log fields, not a client wire.", + "internal/upstream/core/connection_oauth.go:(m).handleOAuthAuthorizationWithResult": "same discovered endpoints.", +} + +// openDoors is the third state a door can be in, and the only honest one for a +// leak that is real, known and being fixed somewhere else: NOT exempted (an +// exemption reads as "a decision was made, and the answer is no rule"), NOT +// routed, but recorded, with the issue that owns it. +// +// Every row here is a live defect. The scan still finds them, so when the fix +// lands the row goes stale and the test says so — the same rule that keeps +// rawServerLeafDoors from rotting. Nothing may be added here to silence a +// finding; a row without an issue number fails. +var openDoors = map[string]string{ + "internal/serveredition/api/user_handlers.go:(m).listServers": "#1161 — the server edition's per-user " + + "server API hands the raw *config.ServerConfig to writeJSON through the ServerResponse wrapper, " + + "publishing env/headers/url/oauth to the authenticated user. Fixed separately.", + "internal/serveredition/api/user_handlers.go:(m).getServer": "#1161 — same wrapper, single-server route.", + "internal/serveredition/api/user_handlers.go:(m).createServer": "#1161 — same wrapper, create echo.", + "internal/serveredition/api/user_handlers.go:(m).updateServer": "#1161 — same wrapper, update echo.", + "internal/serveredition/api/user_handlers.go:(m).enableServer": "#1161 — same wrapper, enable/disable echo. " + + "Found by the round-9 widening, and NOT among the four handlers the issue names: the shape is " + + "identical and it needs the same fix.", +} + +// routedDoors is the other half of the inventory: the doors that DO put a +// server-derived string on a client-facing wire and route it through the shared +// rules. The raw-leaf scan cannot see them — that is the point — so they are +// pinned here instead: the function must still exist, and its body must still +// call one of the shared rules. Deleting the redaction call from any of them +// fails this test even when the leak it would create is invisible to the scan. +var routedDoors = map[string]string{ + // MCP tool responses + "internal/server/mcp.go:(m).handleListUpstreams": "upstream_servers list — headers/env/url/args/command from " + + "redactedServerView, isolation + global docker_isolation from the shared walk, last_error via scrubUpstreamText", + "internal/server/mcp.go:(m).handleAddServerFromRegistry": "add_from_registry echo — command/url from redactedServerView, args via liveRedaction.Argv", + "internal/server/mcp.go:(m).handleListRegistries": "list_registries — registry source url via liveRedaction.URLValue", + "internal/server/mcp.go:(m).createDetailedErrorResponse": "MCP error payload — server_url via liveRedaction.URLValue; error/response_body/error_data scrubbed", + "internal/server/mcp.go:(m).handleListQuarantinedUpstreams": "quarantine_security list_quarantined — the " + + "ORIGINAL #1148 door. Whole server blocks via redactedServerViews (the shared walk).", + "internal/server/mcp.go:(m).handleInspectQuarantinedTools": "quarantine_security inspect — the analysis " + + "echoes the upstream's own text; scrubbed with scrubUpstreamText.", + "internal/server/mcp.go:(m).handleQuarantineSecurity": "the quarantine_security dispatcher — records its own mutation payload through scrubUpstreamTextForAudit.", + "internal/server/mcp.go:(m).handleUpstreamServers": "the upstream_servers dispatcher — every add/update/patch payload is recorded through scrubUpstreamTextForAudit.", + "internal/server/mcp.go:(m).handleUpdateUpstream": "upstream_servers update — the error it reports carries the upstream URL; scrubUpstreamText.", + "internal/server/mcp.go:(m).handleTailLog": "tail_log — upstream log lines carry whatever the server printed; scrubUpstreamLines/scrubUpstreamText.", + "internal/httpapi/server.go:(m).handleGetDiagnostics": "GET /api/v1/diagnostics — the same upstream error prose via oauth.ScrubUpstreamText.", + + // REST — the whole-config door and its write twins (round 9) + "internal/httpapi/server.go:(m).handleGetConfig": "GET /api/v1/config — the WHOLE config " + + "(every server's env/headers/url/oauth plus the global docker_isolation.extra_args and the api_key) " + + "through oauth.RedactedConfig, reveal-gated, failing closed when the walk cannot round-trip", + "internal/httpapi/server.go:(m).handleApplyConfig": "POST /api/v1/config/apply — the write twin of the " + + "above: oauth.UnmaskLiveConfigDocument reverts what binds to a key and REFUSES what does not, " + + "before anything is typed or persisted", + "internal/httpapi/server.go:(m).handlePatchConfig": "PATCH /api/v1/config — oauth.UnmaskLiveConfigTree " + + "resolves the patch against the stored config BEFORE the deep merge, so an echoed mask is reverted " + + "or refused instead of being written over the credential", + "internal/httpapi/import.go:(m).runImport": "POST /api/v1/servers/import{,/json,/path} preview — " + + "url/command from oauth.RedactedConfigView, args via LiveRedaction.Argv, in parity with the CLI twin", + + // REST + "internal/httpapi/server.go:(m).handleAddFromRegistry": "POST registry-add echo — command/url from oauth.RedactedConfigView, args via LiveRedaction.Argv", + "internal/httpapi/server.go:(m).handleAddRegistrySource": "registry source echo via redactedRegistrySummary", + "internal/httpapi/server.go:(m).handleEditRegistrySource": "registry source echo via redactedRegistrySummary", + "internal/httpapi/server.go:(m).handleRemoveRegistrySource": "registry source echo via redactedRegistrySummary", + "internal/httpapi/server.go:(m).handleGetDockerStatus": "GET /api/v1/docker/status — last_error via oauth.ScrubUpstreamText", + + // server edition + "internal/serveredition/api/admin_handlers.go:(m).listAdminServers": "GET /admin/servers — contracts.Server via oauth.RedactServerSecretFields; config-only fallbacks via oauth.RedactedConfigViews", + "internal/serveredition/api/admin_handlers.go:(m).toggleSharedServer": "PUT /admin/servers/{name}/shared echo via oauth.RedactedConfigView", + + // registry catalogue + CLI + diagnostics + "internal/runtime/runtime.go:(m).ListRegistries": "registry source url/servers_url via oauth.LiveRedaction.URLValue", + "cmd/mcpproxy/upstream_cmd.go:buildImportedServersOutput": "`upstream import` preview — url/command from oauth.RedactedConfigView, args via LiveRedaction.Argv", + "internal/upstream/core/monitoring.go:(m).GetConnectionDiagnostics": "command/args/docker_args via oauth.LiveRedaction", + "internal/httpapi/diagnostics_per_server.go:redactHealthDetail": "GET /api/v1/servers/{id}/diagnostics — health.detail via oauth.ScrubUpstreamText", + "internal/httpapi/diagnostics_per_server.go:redactDiagnosticCause": "same route — diagnostic.cause via oauth.ScrubUpstreamText", + "internal/oauth/serverfields.go:RedactServerSecretFields": "the REST + SSE door onto contracts.Server; " + + "the shared rule itself, built entirely from LiveRedaction", + "internal/runtime/event_bus.go:(m).redactServerSecrets": "/events servers.changed — oauth.RedactServerSecretFields", + "internal/httpapi/server.go:redactServerSecretFields": "GET /api/v1/servers and its children — oauth.RedactServerSecretFields", + "internal/upstream/manager.go:(m).GetStats": "upstream_stats fallback — oauth.RedactUpstreamStatsEntry, reveal-gated", + "internal/management/diagnostics.go:(m).Doctor": "doctor UpstreamErrors.ErrorMessage via oauth.ScrubUpstreamText", +} + +// sharedRuleTokens are the identifiers that mean "this function routed its +// payload through the shared redaction rules". +var sharedRuleTokens = []string{ + "LiveRedaction", "liveRedaction", "auditRedaction", + "RedactServerSecretFields", "RedactedConfigView", "RedactedConfigViews", + "RedactUpstreamStats", "RedactUpstreamStatsEntry", + "ScrubUpstreamText", "scrubUpstreamText", "scrubUpstreamTextForAudit", + "redactedServerView", "redactedArgs", "redactedRegistrySummary", + "RedactedConfig", "UnmaskLiveConfigTree", "UnmaskLiveConfigDocument", "viewString", + "isolationValues", "globalIsolationView", "MaskDetectedSecrets", +} + +// doorPackages are the trees scanned for doors: everything that can build a +// client-facing payload out of a server config. +var doorPackages = []string{ + "internal/server", + "internal/httpapi", + "internal/runtime", + "internal/management", + "internal/upstream", + "internal/tray", + "internal/serveredition", + "cmd", +} + +// secretLeafFields are the field names that carry an operator-supplied, +// possibly credential-bearing string on config.ServerConfig / contracts.Server +// and their nested blocks — plus the whole-struct names, so a door that embeds +// a raw config rather than reading its leaves is caught too. +var secretLeafFields = map[string]bool{ + "URL": true, "Env": true, "Headers": true, "Args": true, "Command": true, + "WorkingDir": true, "OAuth": true, "ExtraArgs": true, "ExtraParams": true, + "Scopes": true, "ClientSecret": true, "LastError": true, + "Detail": true, "Cause": true, "Isolation": true, "IsolationDefaults": true, + "ErrorMessage": true, + // whole-struct embeds published as a value: `m["isolation"] = cfg.DockerIsolation` + "DockerIsolation": true, +} + +// redactionCalls are the shared rules and their guards. A value handed to one +// of them is not on a wire, so the scan does not descend into the call. +var redactionCalls = map[string]bool{ + "CheckServerWriteMasks": true, "RedactedConfigView": true, "RedactedConfigViews": true, + "RedactServerSecretFields": true, "RedactUpstreamStats": true, "RedactUpstreamStatsEntry": true, + "ScrubUpstreamText": true, "scrubUpstreamText": true, "scrubUpstreamTextForAudit": true, + "scrubbedConnectionStatus": true, "scrubUpstreamLines": true, + "redactedServerView": true, "redactedServerViews": true, "redactedArgs": true, + "redactedRegistrySummary": true, "isolationValues": true, "globalIsolationView": true, + "RedactedConfig": true, "UnmaskLiveConfigTree": true, "UnmaskLiveConfigDocument": true, + "redactValueWith": true, "redactActivityValue": true, "NormalizeForRedaction": true, + "normalizeForRedaction": true, "FindMaskMarker": true, "ContainsMaskMarker": true, + "URLValue": true, "Leaf": true, "Argv": true, "EnvValue": true, "HeaderValue": true, + "viewField": true, "viewString": true, "viewOr": true, +} + +// publishSinks are the calls that put a value on a wire. +var publishSinks = map[string]bool{ + "Marshal": true, "MarshalIndent": true, "Encode": true, + "NewToolResultText": true, "NewToolResultError": true, + "writeSuccess": true, "writeError": true, "writeJSON": true, + "Publish": true, "Write": true, +} + +func TestNoDoorPublishesARawServerLeaf(t *testing.T) { + found := scanServerPayloadDoors(t) + require.NotEmpty(t, found, "the door scan found nothing — it has stopped testing anything") + + for _, key := range sortedKeys(found) { + if issue, open := openDoors[key]; open { + assert.Contains(t, issue, "#", + "openDoors records %s with no issue number. A row here is a live defect somebody owns; "+ + "without an owner it is an exemption wearing a different name.", key) + continue + } + reason, ok := rawServerLeafDoors[key] + if !assert.True(t, ok, + "NEW DOOR: %s puts an unredacted %s into a response payload.\n"+ + "Route it through the shared rules (oauth.LiveRedaction / RedactServerSecretFields / "+ + "RedactedConfigView / RedactUpstreamStats / ScrubUpstreamText), or — if it genuinely needs "+ + "none — record it in rawServerLeafDoors with the reason. Seven review rounds of issue #1148 "+ + "were one door fixed and the next one left open; this is the check that stops that.", + key, found[key]) { + continue + } + assert.NotEmpty(t, reason, "%s is exempted with no reason recorded", key) + } + + for key := range rawServerLeafDoors { + _, ok := found[key] + assert.True(t, ok, + "rawServerLeafDoors exempts %s, which the scan no longer finds. Delete the row — a stale "+ + "exemption reads as a decision that was made, about code that is gone.", key) + } + + for key := range openDoors { + _, ok := found[key] + assert.True(t, ok, + "openDoors records %s as a KNOWN OPEN LEAK, and the scan no longer finds it — the fix landed. "+ + "Delete the row (or move it to routedDoors) so the next reader is not told a closed door "+ + "is still open.", key) + } +} + +// The other half of the inventory. A routed door is INVISIBLE to the raw-leaf +// scan by construction, so removing its redaction call would silently reopen +// the leak. Each one is pinned to the source: the function must still exist, +// and it must still reach a shared rule. +func TestRoutedDoors_StillCallASharedRule(t *testing.T) { + bodies := indexFunctionBodies(t) + for _, key := range sortedKeys(routedDoors) { + body, ok := bodies[key] + if !assert.True(t, ok, "routedDoors records %s, which no longer exists — delete the row or fix the name", key) { + continue + } + // The row's own prose NAMES the rules the door routes through, and + // each one named must still be there. Round 9: matching "any shared + // rule token anywhere in the body" was too weak to be a guard — + // deleting `redactedServerViews` from handleListQuarantinedUpstreams + // left the test green, because the same body still mentioned + // `liveRedaction` in an unrelated argument. The documentation is now + // the assertion. + required := rulesNamedIn(routedDoors[key]) + if !assert.NotEmpty(t, required, + "routedDoors records %s with prose that names no shared rule, so nothing about it is checked. "+ + "Name the rule(s) it routes through — that sentence IS the assertion.", key) { + continue + } + for _, rule := range required { + assert.Contains(t, body, rule, + "%s no longer calls %s.\n%s\n"+ + "It publishes server-derived strings, so removing the rule republishes them in the clear — "+ + "and the raw-leaf scan cannot see it, because the leaf reaches the payload through a local.", + key, rule, routedDoors[key]) + } + } +} + +// Round 9 finding 4. routedDoors is the half of the inventory the raw-leaf scan +// CANNOT see, so nothing kept it complete: it was written from the +// implementer's prose and it omitted several doors that prose itself lists as +// routed — including handleListQuarantinedUpstreams, the ORIGINAL #1148 door. +// A pin list with holes is worse than none, because the holes are invisible: +// a door missing from it can have its redaction deleted later with nothing +// going red. +// +// The two halves can no longer drift. A ROUTED DOOR is derivable from the +// source — a function that calls one of the shared rules AND reaches a publish +// sink — and every one the scan derives must be pinned here. +func TestRoutedDoors_AreCompleteAgainstTheSource(t *testing.T) { + derived := scanSharedRuleDoors(t) + require.NotEmpty(t, derived, "the routed-door scan found nothing — it has stopped testing anything") + + for _, key := range sortedKeys(derived) { + _, ok := routedDoors[key] + assert.True(t, ok, + "UNPINNED ROUTED DOOR: %s calls a shared redaction rule and publishes the result, but no row in "+ + "routedDoors pins it. The raw-leaf scan is blind to it BY CONSTRUCTION — the leaf reaches the "+ + "payload through a local — so deleting its redaction call would reopen the leak in silence. "+ + "Add the row.", key) + } +} + +// scanSharedRuleDoors derives the routed doors: functions in the door packages +// whose body reaches BOTH a shared redaction rule and a publish sink. +func scanSharedRuleDoors(t *testing.T) map[string]string { + t.Helper() + out := map[string]string{} + bodies := indexFunctionBodies(t) + forEachSourceFile(t, doorPackages, func(_, rel string, fset *token.FileSet, file *ast.File) { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + name := fn.Name.Name + if fn.Recv != nil { + name = "(m)." + name + } + key := rel + ":" + name + body, ok := bodies[key] + if !ok { + continue + } + rule := "" + for _, token := range sharedRuleTokens { + if strings.Contains(body, token) { + rule = token + break + } + } + if rule == "" { + continue + } + publishes := false + ast.Inspect(fn.Body, func(n ast.Node) bool { + if call, ok := n.(*ast.CallExpr); ok && publishSinks[calleeName(call.Fun)] { + publishes = true + } + return true + }) + if publishes { + out[key] = rule + } + } + _ = fset + }) + return out +} + +// rulesNamedIn returns the shared-rule tokens a routedDoors row's prose names. +func rulesNamedIn(why string) []string { + var named []string + for _, token := range sharedRuleTokens { + if strings.Contains(why, token) { + named = append(named, token) + } + } + return named +} + +// indexFunctionBodies renders every top-level function in the door packages +// (plus internal/oauth itself) keyed the same way the scan keys them. +func indexFunctionBodies(t *testing.T) map[string]string { + t.Helper() + root := repoRoot(t) + out := map[string]string{} + for _, pkg := range append(append([]string{}, doorPackages...), "internal/oauth") { + err := filepath.WalkDir(filepath.Join(root, pkg), func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, 0) + if perr != nil { + return nil + } + rel, _ := filepath.Rel(root, path) + rel = filepath.ToSlash(rel) + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + name := fn.Name.Name + if fn.Recv != nil { + name = "(m)." + name + } + var buf bytes.Buffer + if err := printer.Fprint(&buf, fset, fn.Body); err != nil { + continue + } + out[rel+":"+name] = buf.String() + } + return nil + }) + require.NoError(t, err, "walking %s", pkg) + } + return out +} + +// --- round 9: the shapes the round-8 scan was blind to ----------------------- +// +// The round-8 scan matched map literals, `contracts.*` literals, map-index +// assignments, and identifiers tainted from an expression whose SOURCE TEXT +// ended in `.Servers`. Round 9 found live doors it could not see, all of them +// one of two shapes: +// +// - a NAMED STRUCT that embeds or holds a raw config value and is handed to a +// publish sink (`writeJSON(w, 200, &ServerResponse{...})`), or that copies +// the leaves out of one (`ImportedServerResponse{URL: imported.Server.URL}` +// on `POST /api/v1/servers/import`, whose CLI twin round 8 redacted), and +// - a whole `*config.Config` reaching a publish sink through a call — +// `contracts.ConvertConfigToContract(cfg)` on `GET /api/v1/config`, which +// serves every server's env, headers, oauth.client_secret and url +// credentials plus the global docker_isolation.extra_args in one response. +// +// A guard that cannot see the shape the real defects have is worse than none, +// because it reads as coverage. So the scan no longer judges by source text: it +// indexes the declared TYPES of the scanned trees and derives, at a fixpoint, +// which named types carry a raw `config.Config` / `config.ServerConfig`, which +// expose a secret-bearing leaf field, which actually REACH a publish sink, and +// which functions RETURN a raw config — all from the same source it scans. + +// configTypeIndex is that derived index. Every set is keyed by the type's +// PACKAGE-QUALIFIED name (`api.ServerResponse`, `config.ServerConfig`): a bare +// name collides across the tree — a dozen packages declare a `Client` — and a +// collision in a fail-closed scan is noise, which is how an inventory stops +// being kept green. +type configTypeIndex struct { + // bearing names the types that transitively hold a raw config value: + // `api.ServerResponse` (it embeds *config.ServerConfig), + // `api.ServerListResponse` (it holds []*api.ServerResponse), and anything + // else built the same way. + bearing map[string]bool + // leafy names the struct types with a secret-bearing leaf field of their + // own — the `httpapi.ImportedServerResponse` shape, a response DTO that + // copies `url` / `command` / `args` out of a config. + leafy map[string]bool + // published names the types a value of which actually REACHES a publish + // sink, derived in two steps below. It is what keeps the widened scan from + // flagging every internal struct that holds a config: an inventory nobody + // keeps green is the same fail-open shape as no inventory at all. + published map[string]bool + // returning names the functions, methods and interface methods whose + // declared results include a raw config value, so an identifier assigned + // from one is known to hold a config without a type checker. + // + // Only UNANIMOUS names qualify: `Load` and `ListServers` are each declared + // a dozen times across this tree, and only some of those return a config, + // so taking any of them would taint an identifier that holds something + // else entirely. Without a type checker the call site cannot tell them + // apart — and a scan whose findings are mostly collisions is a scan whose + // exemption table stops being read. + returning map[string]bool + // fieldsOf and results are the graph the derivations walk. Field type + // expressions are stored with the package they were written in, since an + // unqualified name in a field means "this package". + fieldsOf map[string][]qualifiedExpr + results map[string][]string + // declared / declaredRawConfig count the declarations behind each function + // name, so `returning` can require unanimity. + declared map[string]int + declaredRawConfig map[string]int + // containerFields are the struct FIELD names declared as a map or slice of + // raw configs (`serverConfigs map[string]*config.ServerConfig`). Writing + // into one is a config write, not a publish — the same judgement + // rawConfigContainers makes for locals, for the fields a function reaches + // through its receiver. + containerFields map[string]bool +} + +// qualifiedExpr is a type expression plus the package it was written in. +type qualifiedExpr struct { + expr ast.Expr + pkg string +} + +// nonCarryingCalls cannot hand a config onward whatever they are given: they +// return a length or a scalar conversion of it. +var nonCarryingCalls = map[string]bool{ + "len": true, "cap": true, "int": true, "int32": true, "int64": true, + "uint": true, "uint32": true, "uint64": true, "float64": true, "bool": true, +} + +// configTypeTrees are the trees whose type declarations are indexed. The door +// packages and internal/contracts contribute types; internal/config contributes +// only function signatures — its own structs are the INPUT shape, not a +// payload, and indexing them would flag every site that builds a ServerConfig +// from a request body. +var configTypeTrees = append(append([]string{}, doorPackages...), "internal/contracts") + +// inConfigTypeTrees reports whether a file contributes STRUCT types to the +// index. internal/config is excluded: its own structs are the INPUT shape, not +// a payload, and indexing them would flag every site that builds a +// ServerConfig from a request body. +func inConfigTypeTrees(rel string) bool { + for _, tree := range configTypeTrees { + if rel == tree || strings.HasPrefix(rel, tree+"/") { + return true + } + } + return false +} + +// rawConfigTypes are the qualified names of a raw, unredacted config value. +var rawConfigTypes = map[string]bool{ + "config.Config": true, "config.ServerConfig": true, +} + +// namedTypesIn reports every named type a type expression references, qualified +// by `selfPkg` when the source wrote it unqualified. It looks through +// pointers, slices, arrays, maps and variadics. +func namedTypesIn(e ast.Expr, selfPkg string, out func(qualified string)) { + switch t := e.(type) { + case *ast.Ident: + out(selfPkg + "." + t.Name) + case *ast.SelectorExpr: + if id, ok := t.X.(*ast.Ident); ok { + out(id.Name + "." + t.Sel.Name) + } + case *ast.StarExpr: + namedTypesIn(t.X, selfPkg, out) + case *ast.ArrayType: + namedTypesIn(t.Elt, selfPkg, out) + case *ast.MapType: + namedTypesIn(t.Key, selfPkg, out) + namedTypesIn(t.Value, selfPkg, out) + case *ast.Ellipsis: + namedTypesIn(t.Elt, selfPkg, out) + } +} + +// indexConfigTypes builds the index by parsing the same trees the scan walks. +func indexConfigTypes(t *testing.T) *configTypeIndex { + t.Helper() + idx := &configTypeIndex{ + bearing: map[string]bool{}, + leafy: map[string]bool{}, + published: map[string]bool{}, + returning: map[string]bool{}, + fieldsOf: map[string][]qualifiedExpr{}, + results: map[string][]string{}, + + declared: map[string]int{}, + declaredRawConfig: map[string]int{}, + containerFields: map[string]bool{}, + } + + // Struct types come from the door packages and internal/contracts; function + // SIGNATURES are censused across the whole tree, because unanimity can only + // be judged against every declaration of a name, not against the subset a + // narrower walk happens to see. + forEachSourceFile(t, []string{"internal", "cmd"}, + func(_, rel string, _ *token.FileSet, file *ast.File) { + selfPkg := file.Name.Name + indexStructs := inConfigTypeTrees(rel) + + recordResults := func(name string, results *ast.FieldList) { + idx.declared[name]++ + if results == nil { + return + } + raw := false + for _, res := range results.List { + namedTypesIn(res.Type, selfPkg, func(q string) { + if rawConfigTypes[q] { + raw = true + } + idx.results[name] = append(idx.results[name], q) + }) + } + if raw { + idx.declaredRawConfig[name]++ + } + } + + ast.Inspect(file, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.FuncDecl: + recordResults(x.Name.Name, x.Type.Results) + case *ast.InterfaceType: + for _, m := range x.Methods.List { + ft, ok := m.Type.(*ast.FuncType) + if !ok || len(m.Names) == 0 { + continue + } + recordResults(m.Names[0].Name, ft.Results) + } + case *ast.TypeSpec: + st, ok := x.Type.(*ast.StructType) + if !ok || !indexStructs || st.Fields == nil { + return true + } + name := selfPkg + "." + x.Name.Name + for _, f := range st.Fields.List { + idx.fieldsOf[name] = append(idx.fieldsOf[name], qualifiedExpr{f.Type, selfPkg}) + for _, fieldName := range f.Names { + if secretLeafFields[fieldName.Name] { + idx.leafy[name] = true + } + if isRawConfigContainerType(f.Type, selfPkg) { + idx.containerFields[fieldName.Name] = true + } + } + if len(f.Names) == 0 { + // An embedded field: its own name is its type's. + namedTypesIn(f.Type, selfPkg, func(q string) { + if secretLeafFields[q[strings.LastIndex(q, ".")+1:]] { + idx.leafy[name] = true + } + }) + } + } + } + return true + }) + }) + + // A type is config-bearing if any field is a raw config value, or is a + // type that is itself config-bearing. Iterate to a fixpoint so the wrapper + // of a wrapper is caught. + for changed := true; changed; { + changed = false + for name, fieldTypes := range idx.fieldsOf { + if idx.bearing[name] { + continue + } + for _, ft := range fieldTypes { + hit := false + namedTypesIn(ft.expr, ft.pkg, func(q string) { + if rawConfigTypes[q] || idx.bearing[q] { + hit = true + } + }) + if hit { + idx.bearing[name] = true + changed = true + break + } + } + } + } + for name, n := range idx.declaredRawConfig { + if n == idx.declared[name] { + idx.returning[name] = true + } + } + idx.derivePublishedTypes(t) + return idx +} + +// derivePublishedTypes finds the named types a value of which actually reaches +// a wire. +// +// Step 1, the SEED: every argument of every publish sink in the door packages +// whose named type the source makes evident — a composite literal, an +// identifier assigned from one or from a call, or a call whose declared result +// type is known. `s.writeSuccess(w, result)` where `result, err := +// s.runImport(...)` seeds `httpapi.ImportResponse`. +// +// Step 2, the CLOSURE: a published type publishes its fields' types too, so +// `httpapi.ImportedServerResponse` is published because ImportResponse holds a +// slice of it. That is the hop the round-8 scan had no way to make, and it is +// why the import preview's `url` / `command` / `args` were invisible to it +// while its CLI twin was being redacted. +func (idx *configTypeIndex) derivePublishedTypes(t *testing.T) { + t.Helper() + forEachSourceFile(t, doorPackages, func(_, _ string, _ *token.FileSet, file *ast.File) { + selfPkg := file.Name.Name + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + locals := idx.localTypes(fn, selfPkg) + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !publishSinks[calleeName(call.Fun)] { + return true + } + for _, arg := range call.Args { + if name, ok := idx.exprTypeName(arg, selfPkg, locals); ok { + idx.published[name] = true + } + } + return true + }) + } + }) + for changed := true; changed; { + changed = false + for name := range idx.published { + for _, ft := range idx.fieldsOf[name] { + namedTypesIn(ft.expr, ft.pkg, func(q string) { + if _, known := idx.fieldsOf[q]; known && !idx.published[q] { + idx.published[q] = true + changed = true + } + }) + } + } + } +} + +// localTypes maps this function's identifiers to the named type they hold, +// where the source says so syntactically. +func (idx *configTypeIndex) localTypes(fn *ast.FuncDecl, selfPkg string) map[string]string { + locals := map[string]string{} + for pass := 0; pass < 2; pass++ { + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.AssignStmt: + for i, lhs := range x.Lhs { + id, ok := lhs.(*ast.Ident) + if !ok || i >= len(x.Rhs) { + continue + } + if name, ok := idx.exprTypeName(x.Rhs[i], selfPkg, locals); ok { + locals[id.Name] = name + } + } + case *ast.ValueSpec: + if x.Type == nil { + return true + } + namedTypesIn(x.Type, selfPkg, func(q string) { + for _, id := range x.Names { + locals[id.Name] = q + } + }) + } + return true + }) + } + return locals +} + +// exprTypeName resolves the qualified named type of an expression from the +// source alone. +func (idx *configTypeIndex) exprTypeName(e ast.Expr, selfPkg string, locals map[string]string) (string, bool) { + switch x := e.(type) { + case *ast.ParenExpr: + return idx.exprTypeName(x.X, selfPkg, locals) + case *ast.UnaryExpr: + if x.Op == token.AND { + return idx.exprTypeName(x.X, selfPkg, locals) + } + case *ast.CompositeLit: + return litTypeName(x, selfPkg) + case *ast.Ident: + if name, ok := locals[x.Name]; ok { + return name, true + } + case *ast.CallExpr: + for _, name := range idx.results[calleeName(x.Fun)] { + if _, known := idx.fieldsOf[name]; known { + return name, true + } + } + } + return "", false +} + +// isPayloadLitType reports whether a named struct type is a response payload: +// a value of it reaches a publish sink (directly, or as a field of something +// that does) AND it carries a secret-bearing leaf field or a raw config of its +// own. +// +// Both halves are needed. Without "reaches a sink" the scan flags every +// internal struct with a `URL` field, including the REQUEST bodies the write +// doors parse; without "carries a leaf" it flags every DTO on the tree. +func (idx *configTypeIndex) isPayloadLitType(name string) bool { + return idx.published[name] && (idx.leafy[name] || idx.bearing[name]) +} + +// forEachSourceFile parses every non-test .go file of the given trees. Build +// tags are deliberately NOT honoured: `internal/serveredition` is behind +// `//go:build server`, and a door that only exists in one edition is still a +// door. +func forEachSourceFile(t *testing.T, trees []string, visit func(pkg, rel string, fset *token.FileSet, file *ast.File)) { + t.Helper() + root := repoRoot(t) + for _, pkg := range trees { + err := filepath.WalkDir(filepath.Join(root, pkg), func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, 0) + if perr != nil { + return nil + } + rel, _ := filepath.Rel(root, path) + visit(pkg, filepath.ToSlash(rel), fset, file) + return nil + }) + require.NoError(t, err, "walking %s", pkg) + } +} + +// scanServerPayloadDoors parses the door packages and returns, per function, +// the secret-bearing leaves it puts into a response map, a payload struct, or a +// publish sink. +func scanServerPayloadDoors(t *testing.T) map[string]string { + t.Helper() + idx := indexConfigTypes(t) + require.NotEmpty(t, idx.bearing, "the type index found no config-bearing type — the widened scan is inert") + require.NotEmpty(t, idx.published, "the type index found no published type — the widened scan is inert") + out := map[string]string{} + + forEachSourceFile(t, doorPackages, func(_, rel string, _ *token.FileSet, file *ast.File) { + selfPkg := file.Name.Name + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + if leaves := scanFuncForPublishedLeaves(fn, selfPkg, idx); len(leaves) > 0 { + name := fn.Name.Name + if fn.Recv != nil { + name = "(m)." + name + } + out[rel+":"+name] = strings.Join(leaves, ", ") + } + } + }) + return out +} + +// scanFuncForPublishedLeaves reports the secret-bearing leaves this function +// puts on a wire. +// +// Four shapes are matched. A SELECTOR on one of secretLeafFields used as a +// value in a response literal, assigned into a map index, or passed to a +// publish sink — that is `serverMap["url"] = cfg.URL`, the shape every leak on +// this issue has had. A bare IDENTIFIER holding a raw config value — from a raw +// config collection, from a function DECLARED to return one, or from a +// `config.Config{}` literal. A COMPOSITE LITERAL of a config-bearing named type +// — `&ServerResponse{ServerConfig: sc}`, which no source-text rule could see. +// And a CALL carrying a raw config into a payload — +// `contracts.ConvertConfigToContract(cfg)` on `GET /api/v1/config`. +func scanFuncForPublishedLeaves(fn *ast.FuncDecl, selfPkg string, idx *configTypeIndex) []string { + rawIdents := rawConfigIdents(fn, selfPkg, idx) + containers := rawConfigContainers(fn, selfPkg) + seen := map[string]bool{} + var leaves []string + + add := func(name string) { + if name == "" || seen[name] { + return + } + seen[name] = true + leaves = append(leaves, name) + } + + var record func(ast.Expr) + record = func(e ast.Expr) { + switch x := e.(type) { + case *ast.ParenExpr: + record(x.X) + case *ast.UnaryExpr: + if x.Op == token.AND { + record(x.X) + } + case *ast.SelectorExpr: + switch { + case secretLeafFields[x.Sel.Name]: + add(x.Sel.Name) + case isRawConfigExpr(renderExpr(x)): + // A whole raw config collection published as a value — + // `writeJSON(w, 200, h.config.Servers)`, the shape the server + // edition's admin API had. + add("raw config collection (" + renderExpr(x) + ")") + } + case *ast.Ident: + if rawIdents[x.Name] { + add("raw config value (" + x.Name + ")") + } + case *ast.CompositeLit: + // Gated on `published` for the same reason isPayloadLitType is: + // plenty of internal structs hold a config and never reach a wire. + // A bearing type newly handed to a sink is published BY that hand- + // off, so the gate costs the scan nothing it needs. + if name, ok := litTypeName(x, selfPkg); ok && idx.bearing[name] && idx.published[name] { + add("named type holding a raw config (" + name + ")") + } + case *ast.CallExpr: + if redactionCalls[calleeName(x.Fun)] || nonCarryingCalls[calleeName(x.Fun)] { + return + } + // A raw config carried into a payload through a call. The call is + // not a redaction rule — the scan checked — so whatever it returns + // still holds the operator's credentials. + for _, arg := range x.Args { + if id, ok := arg.(*ast.Ident); ok && rawIdents[id.Name] { + add("raw config value (" + calleeName(x.Fun) + "(" + id.Name + "))") + } + } + } + } + + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.CompositeLit: + if !isResponseLit(x, selfPkg, idx) { + return true + } + for _, el := range x.Elts { + if kv, ok := el.(*ast.KeyValueExpr); ok { + record(kv.Value) + continue + } + record(el) + } + case *ast.AssignStmt: + for i, lhs := range x.Lhs { + ix, ok := lhs.(*ast.IndexExpr) + if !ok || i >= len(x.Rhs) { + continue + } + // `updated.Servers[i] = stamped` and + // `storedServerMap[name] = storedServer` write INTO a config or + // a map of them; they do not publish one. Only the response-map + // shape (`payload["url"] = …`) is a door. + base := renderExpr(ix.X) + if strings.HasSuffix(base, ".Servers") || containers[base] || + idx.containerFields[base[strings.LastIndex(base, ".")+1:]] { + continue + } + record(x.Rhs[i]) + } + case *ast.CallExpr: + callee := calleeName(x.Fun) + if redactionCalls[callee] { + // The payload is BEING redacted (or checked). Its arguments are + // not on a wire; descending would flag the fix as the defect. + return false + } + if !publishSinks[callee] { + return true + } + for _, arg := range x.Args { + record(arg) + } + } + return true + }) + sort.Strings(leaves) + return leaves +} + +// litTypeName returns the package-qualified name of a composite literal's type. +func litTypeName(x *ast.CompositeLit, selfPkg string) (string, bool) { + switch t := x.Type.(type) { + case *ast.Ident: + return selfPkg + "." + t.Name, true + case *ast.SelectorExpr: + if id, ok := t.X.(*ast.Ident); ok { + return id.Name + "." + t.Sel.Name, true + } + } + return "", false +} + +// rawConfigIdents returns the local identifiers that hold a RAW config value. +// +// Four sources, each derived rather than asserted: an expression that renders +// as `.Servers`; a call to a function whose DECLARED results +// include a `config.Config` / `config.ServerConfig` — that is what makes +// `cfg, err := s.desiredConfigForPatch()` visible without a type checker; a +// `config.Config{}` literal or a `var cfg config.Config` declaration, which is +// how the config WRITE doors receive one; and one hop of aliasing or ranging +// off any of those, so `for _, sc := range h.config.Servers { found = sc }` +// taints `found` too. +func rawConfigIdents(fn *ast.FuncDecl, selfPkg string, idx *configTypeIndex) map[string]bool { + tainted := map[string]bool{} + isRawSource := func(e ast.Expr) bool { + switch x := e.(type) { + case *ast.UnaryExpr: + if x.Op == token.AND { + return isRawConfigLit(x.X, selfPkg) + } + case *ast.CompositeLit: + return isRawConfigLit(x, selfPkg) + case *ast.CallExpr: + if idx.returning[calleeName(x.Fun)] { + return true + } + } + rendered := renderExpr(e) + return isRawConfigExpr(rendered) || tainted[rendered] + } + redacted := map[string]bool{} + for pass := 0; pass < 2; pass++ { + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.CallExpr: + // A value handed to a shared rule has been masked IN PLACE. + // `s.redactServerSecrets(serverValues)` is the whole reason + // GET /api/v1/servers is safe, so the identifier it masked is + // no longer a raw config on the wire. + if redactionCalls[calleeName(x.Fun)] { + for _, arg := range x.Args { + if id, ok := arg.(*ast.Ident); ok { + redacted[id.Name] = true + } + } + } + case *ast.AssignStmt: + for i, lhs := range x.Lhs { + id, ok := lhs.(*ast.Ident) + if !ok || i >= len(x.Rhs) { + continue + } + // `published = oauth.RedactedConfig(cfg)` — the identifier + // now holds the MASKED copy, whatever it held before. + if call, ok := x.Rhs[i].(*ast.CallExpr); ok && redactionCalls[calleeName(call.Fun)] { + redacted[id.Name] = true + continue + } + if isRawSource(x.Rhs[i]) { + tainted[id.Name] = true + } + } + case *ast.ValueSpec: + for _, id := range x.Names { + if x.Type != nil && isRawConfigTypeExpr(x.Type, selfPkg) { + tainted[id.Name] = true + } + } + case *ast.RangeStmt: + if x.Value != nil && isRawSource(x.X) { + if id, ok := x.Value.(*ast.Ident); ok { + tainted[id.Name] = true + } + } + } + return true + }) + } + for name := range redacted { + delete(tainted, name) + } + delete(tainted, "_") + return tainted +} + +// isRawConfigContainerType reports whether a type expression is a map or slice +// of raw configs. +func isRawConfigContainerType(e ast.Expr, selfPkg string) bool { + switch t := e.(type) { + case *ast.MapType: + return isRawConfigTypeExpr(t.Value, selfPkg) + case *ast.ArrayType: + return isRawConfigTypeExpr(t.Elt, selfPkg) + } + return false +} + +// rawConfigContainers returns the rendered expressions that name a MAP or +// SLICE of raw configs — `storedServerMap := make(map[string]*config.ServerConfig)`, +// `w.servers = make(map[string]*config.ServerConfig, n)`. +// +// Writing into one of those is a config write, not a publish, so the +// response-map rule (`payload["url"] = cfg.URL`) must not fire on it. The +// container is identified from the source that BUILDS it, so a map nobody can +// see the type of still counts as a response map and still fails closed. +func rawConfigContainers(fn *ast.FuncDecl, selfPkg string) map[string]bool { + containers := map[string]bool{} + isContainerType := func(e ast.Expr) bool { return isRawConfigContainerType(e, selfPkg) } + isContainerSource := func(e ast.Expr) bool { + switch x := e.(type) { + case *ast.CallExpr: + if id, ok := x.Fun.(*ast.Ident); ok && id.Name == "make" && len(x.Args) > 0 { + return isContainerType(x.Args[0]) + } + case *ast.CompositeLit: + return x.Type != nil && isContainerType(x.Type) + } + return false + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.AssignStmt: + for i, lhs := range x.Lhs { + if i < len(x.Rhs) && isContainerSource(x.Rhs[i]) { + containers[renderExpr(lhs)] = true + } + } + case *ast.ValueSpec: + if x.Type != nil && isContainerType(x.Type) { + for _, id := range x.Names { + containers[id.Name] = true + } + } + } + return true + }) + return containers +} + +// isRawConfigTypeExpr reports whether a type expression names a raw config value. +func isRawConfigTypeExpr(e ast.Expr, selfPkg string) bool { + raw := false + namedTypesIn(e, selfPkg, func(q string) { + if rawConfigTypes[q] { + raw = true + } + }) + return raw +} + +// isRawConfigLit reports whether a composite literal builds a raw config value. +func isRawConfigLit(e ast.Expr, selfPkg string) bool { + lit, ok := e.(*ast.CompositeLit) + if !ok || lit.Type == nil { + return false + } + return isRawConfigTypeExpr(lit.Type, selfPkg) +} + +// isRawConfigExpr reports whether a rendered expression names a raw config +// server collection — `h.config.Servers`, `cfg.Servers`, `currentConfig.Servers`. +func isRawConfigExpr(rendered string) bool { + return strings.HasSuffix(rendered, ".Servers") && strings.Contains(strings.ToLower(rendered), "config") +} + +// renderExpr prints an expression back to source text. +func renderExpr(e ast.Expr) string { + var buf bytes.Buffer + if err := printer.Fprint(&buf, token.NewFileSet(), e); err != nil { + return "" + } + return buf.String() +} + +// isResponseLit reports whether a composite literal is a response payload: a +// map, a contracts.* struct, a payload struct type (round 9), or a nested +// element of one. +// +// The payload-struct arm is round 9's widening. `ImportedServerResponse{URL: +// …}` — the import preview's response DTO — is a plain named struct, so the +// round-8 rule (map / contracts / nested) walked straight past it while its +// CLI twin was being redacted. +func isResponseLit(x *ast.CompositeLit, selfPkg string, idx *configTypeIndex) bool { + if x.Type == nil { + return true + } + if _, ok := x.Type.(*ast.MapType); ok { + return true + } + if t, ok := x.Type.(*ast.SelectorExpr); ok { + if id, ok := t.X.(*ast.Ident); ok && id.Name == "contracts" { + return true + } + } + name, ok := litTypeName(x, selfPkg) + return ok && idx.isPayloadLitType(name) +} + +func calleeName(fun ast.Expr) string { + switch f := fun.(type) { + case *ast.Ident: + return f.Name + case *ast.SelectorExpr: + return f.Sel.Name + } + return "" +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// repoRoot walks up from this package to the directory holding go.mod. +func repoRoot(t *testing.T) string { + t.Helper() + dir := packageDir(t) + for i := 0; i < 8; i++ { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + dir = filepath.Dir(dir) + } + t.Fatal("could not locate the repository root (no go.mod above " + packageDir(t) + ")") + return "" +} diff --git a/internal/oauth/round8_one_answer_test.go b/internal/oauth/round8_one_answer_test.go new file mode 100644 index 000000000..2284b2d4b --- /dev/null +++ b/internal/oauth/round8_one_answer_test.go @@ -0,0 +1,101 @@ +package oauth + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" +) + +// Issue #1148, round 8 finding 3. `command` and `working_dir` are ONE leaf with +// THREE doors and, before this round, TWO answers: the generic MCP view +// (`quarantine_security list_quarantined`, built by the shared walk) ran the +// leaf rule over them, while `upstream_servers list` and +// RedactServerSecretFields (REST + SSE) published them RAW — and the decision +// table recorded them as not-secret, which is the third answer. +// +// A command path is rarely a credential. That is a reason for the rule to leave +// it readable, not a reason for three doors to disagree: the shared leaf rule +// already passes `npx` and `/opt/tools` through untouched, and it is the only +// thing that also catches a token pasted into either field. +func TestCommandAndWorkingDir_HaveOneAnswerOnEveryDoor(t *testing.T) { + const token = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + + // The generic MCP view — the reference answer. + view := RedactedConfigView("", &config.ServerConfig{ + Name: "s", Command: token, WorkingDir: "/opt/" + token, + }) + require.NotNil(t, view) + + srv := &contracts.Server{Name: "s", Command: token, WorkingDir: "/opt/" + token} + RedactServerSecretFields(srv) + + assert.NotContains(t, srv.Command, token, + "the REST/SSE door published `command` verbatim while the MCP door masks it") + assert.NotContains(t, srv.WorkingDir, token, + "the REST/SSE door published `working_dir` verbatim while the MCP door masks it") + assert.Equal(t, view["command"], srv.Command, "one leaf, one answer") + assert.Equal(t, view["working_dir"], srv.WorkingDir, "one leaf, one answer") +} + +// The readable half of the same rule: an ordinary command and an ordinary +// working directory must survive every door byte-identical, or masking them +// would have cost the operator the field. +func TestCommandAndWorkingDir_StayReadableWhenTheyCarryNoCredential(t *testing.T) { + srv := &contracts.Server{Name: "s", Command: "npx", WorkingDir: "/opt/tools/mcp"} + RedactServerSecretFields(srv) + assert.Equal(t, "npx", srv.Command) + assert.Equal(t, "/opt/tools/mcp", srv.WorkingDir) +} + +// The decision table is the third door: it recorded `command` / `working_dir` +// as not-secret, which is what let two of the three implementations disagree +// without any guard noticing. +func TestCommandAndWorkingDir_DecisionMatchesTheRule(t *testing.T) { + for _, path := range []string{"command", "working_dir"} { + assert.Equal(t, MaskDecisionRefuse, ServerFieldMaskDecisions[path], + "%s is masked on every read door and no write rule binds a revert to it, "+ + "so an echoed mask must be refused", path) + } +} + +// RedactedConfigView is the hoisted view every door can now reach — including +// the ones below internal/server in the import graph. +func TestRedactedConfigView_MasksEverySecretBearingLeaf(t *testing.T) { + const token = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + view := RedactedConfigView("", &config.ServerConfig{ + Name: "s", + URL: "https://h/mcp?opaque=" + token, + Env: map[string]string{"BENIGN": token}, + Headers: map[string]string{"X-Custom": token}, + Args: []string{"--endpoint=" + token}, + OAuth: &config.OAuthConfig{ClientSecret: token}, + Isolation: &config.IsolationConfig{ExtraArgs: []string{"-e", "API_KEY=" + token}}, + }) + require.NotNil(t, view) + assert.NotContains(t, flatten(view), token, "the shared view published a credential: %#v", view) +} + +func flatten(v interface{}) string { + switch typed := v.(type) { + case map[string]interface{}: + out := "" + for k, val := range typed { + out += k + "=" + flatten(val) + ";" + } + return out + case []interface{}: + out := "" + for _, val := range typed { + out += flatten(val) + "," + } + return out + case string: + return typed + default: + return "" + } +} diff --git a/internal/oauth/round8_renderer_discovery_test.go b/internal/oauth/round8_renderer_discovery_test.go new file mode 100644 index 000000000..7a7575bd1 --- /dev/null +++ b/internal/oauth/round8_renderer_discovery_test.go @@ -0,0 +1,318 @@ +package oauth + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Issue #1148, round 8 finding 4. +// +// Round 7 replaced a hand-maintained MARKER list with a hand-maintained +// RENDERER list (maskRenderings) and derived the markers from it. That removed +// one way to fall behind and left the other: nothing failed when a renderer was +// ADDED to this package and not registered, so the fail-closed net would keep +// accepting an echo of a rendering it had never been told about — which is +// exactly the shape of round 7's own defect (`***REDACTED***`). +// +// The gate below closes it structurally. The package's own SOURCE is parsed to +// discover every exported `func(string) string` — every function shaped like a +// mask rendering — and the test fails unless each discovered name is either +// BOUND into exportedStringFuncs (where its output is probed against the net) +// or recorded in notMaskRenderings with a reason. Adding a function to this +// package therefore cannot leave the net behind: the discovery set changes and +// this test goes red in the same edit. + +// exportedStringFuncs binds every exported `func(string) string` in this +// package that renders or rewrites a value. TestExportedStringFuncs_AreAllBound +// fails when the package grows one that is missing here. +var exportedStringFuncs = map[string]func(string) string{ + "MaskValue": MaskValue, + "AuditMaskValue": AuditMaskValue, + "MaskDetectedSecrets": MaskDetectedSecrets, + "RedactSensitiveData": RedactSensitiveData, + "RedactURL": RedactURL, + "RedactURLQueryParams": RedactURLQueryParams, + "ScrubUpstreamText": ScrubUpstreamText, + "ArgvFlagKey": ArgvFlagKey, +} + +// Round 9 finding 5. Discovery covered PACKAGE-LEVEL functions only, and the +// renderings that actually reach the live read doors are METHODS on Redaction: +// `Leaf`, `EnvValue`, `HeaderValue`, `URLValue` and `Argv` are what +// RedactServerSecretFields, the MCP view and the generic walk all call, and +// they were hand-listed in maskRenderings with nothing enforcing registration +// — the exact hole round 8 closed for functions and left open for the half +// that matters. +// +// redactionMethodRenderers binds each one, under both policies and under both +// a sensitive and a benign key, so the property below probes the rendering a +// read door actually publishes. +var redactionMethodRenderers = map[string][]func(string) string{ + "Redaction.Leaf": { + func(s string) string { return LiveRedaction.Leaf("token", s) }, + func(s string) string { return LiveRedaction.Leaf("benign", s) }, + func(s string) string { return LiveRedaction.Leaf("url", s) }, + func(s string) string { return AuditRedaction.Leaf("token", s) }, + }, + "Redaction.EnvValue": { + func(s string) string { return LiveRedaction.EnvValue("API_KEY", s) }, + func(s string) string { return LiveRedaction.EnvValue("BENIGN", s) }, + func(s string) string { return AuditRedaction.EnvValue("API_KEY", s) }, + }, + "Redaction.HeaderValue": { + func(s string) string { return LiveRedaction.HeaderValue("Authorization", s) }, + func(s string) string { return LiveRedaction.HeaderValue("X-Trace", s) }, + func(s string) string { return AuditRedaction.HeaderValue("Authorization", s) }, + }, + "Redaction.URLValue": { + func(s string) string { return LiveRedaction.URLValue(s) }, + func(s string) string { return AuditRedaction.URLValue(s) }, + }, + "Redaction.Argv": { + // A flag/value pair and a lone positional token: the two shapes the + // argv rule renders differently. + func(s string) string { return strings.Join(LiveRedaction.Argv([]string{"--token", s}), " ") }, + func(s string) string { return strings.Join(LiveRedaction.Argv([]string{s}), " ") }, + func(s string) string { return strings.Join(AuditRedaction.Argv([]string{"--token", s}), " ") }, + }, +} + +// notMaskRenderings records the discovered renderers that are NOT mask +// renderings, with the reason. A row here is a deliberate decision, and a stale +// one (naming something that no longer exists) fails the test — the same rule +// the field decision table lives by. +var notMaskRenderings = map[string]string{ + "ExtractResourceMetadataURL": "parses a WWW-Authenticate header and returns the resource-metadata URL it advertises; " + + "it extracts, it does not mask, and its output never reaches a write door as an echoed value", + "Redaction.CapString": "a property of the DESTINATION, not a rule: it caps a leaf for a PERSISTED surface " + + "(the activity store) and introduces no mask of its own. The live read policies set no limit precisely " + + "so a masked value is never truncated below what the write path can recognise, which is why binding it " + + "here would probe an identity function and prove nothing.", +} + +// maskProbes are the value shapes a read door actually carries: bare secrets, +// keyed secrets, connection strings, URLs with sensitive and unrecognised +// parameters, a vendor-formatted token, and two ordinary values that must +// survive untouched. +var maskProbes = []string{ + "supersecretvalue", + "abcd", + "API_KEY=supersecretvalue", + "Authorization: Bearer supersecretvalue", + "postgres://u:supersecretvalue@host/db", + "https://host/mcp?access_token=supersecretvalue", + "https://host/mcp?opaque=ghp_1234567890abcdefghijABCDEFGHIJ123456", + "AKIAIOSFODNN7EXAMPLE", + "npx", + "/opt/tools/mcp", +} + +func TestExportedStringFuncs_AreAllBound(t *testing.T) { + discovered := discoverExportedStringFuncs(t) + require.NotEmpty(t, discovered, "the AST discovery found nothing — it has stopped testing anything") + require.Contains(t, discovered, "Redaction.Leaf", + "discovery no longer reaches the Redaction METHODS — the renderings the live read doors actually "+ + "publish. That is the half round 9 found unenforced; a discovery that cannot see them reads as "+ + "coverage without being any.") + + for _, name := range discovered { + _, bound := exportedStringFuncs[name] + if _, boundMethod := redactionMethodRenderers[name]; boundMethod { + bound = true + } + _, exempt := notMaskRenderings[name] + assert.True(t, bound || exempt, + "oauth.%s is an exported func(string) string — the shape of a mask rendering — and is "+ + "registered nowhere. Bind it into exportedStringFuncs (so its output is probed against "+ + "the fail-closed net) or record it in notMaskRenderings with a reason. A rendering the "+ + "net does not know about is accepted on the write path and PERSISTED over the credential.", + name) + } + + known := map[string]bool{} + for _, name := range discovered { + known[name] = true + } + for name := range exportedStringFuncs { + assert.True(t, known[name], "exportedStringFuncs binds oauth.%s, which no longer exists", name) + } + for name := range redactionMethodRenderers { + assert.True(t, known[name], "redactionMethodRenderers binds oauth.%s, which no longer exists", name) + } + for name := range notMaskRenderings { + assert.True(t, known[name], "notMaskRenderings exempts oauth.%s, which no longer exists", name) + } +} + +// And the property the binding exists for: whenever a bound renderer REWRITES a +// value, the fail-closed net must recognise the result. A renderer that emits a +// marker MaskMarkers does not carry is a mask a write door would accept. +func TestExportedStringFuncs_RenderOnlyRecognisedMasks(t *testing.T) { + check := func(name, probe, got string) { + if got == probe { + return // pass-through: nothing was masked, nothing to recognise + } + assert.True(t, ContainsMaskMarker(got), + "oauth.%s(%q) = %q, which carries no marker the fail-closed net recognises. "+ + "Register the rendering in maskRenderings so MaskMarkers is derived from it, "+ + "or the net will accept this string on a write and persist it over the credential.", + name, probe, got) + } + for name, fn := range exportedStringFuncs { + for _, probe := range maskProbes { + check(name, probe, fn(probe)) + } + } + // The half that matters: the METHODS the live read doors publish through. + for name, renderers := range redactionMethodRenderers { + for _, render := range renderers { + for _, probe := range maskProbes { + check(name, probe, render(probe)) + } + } + } +} + +// discoverExportedStringFuncs parses this package's own source and returns the +// names of every exported top-level `func(string) string`, PLUS every exported +// method on Redaction that renders a value — one `string` or `[]string` result +// (round 9 finding 5). Methods come back as `Redaction.` so the two +// namespaces cannot collide. +func discoverExportedStringFuncs(t *testing.T) []string { + t.Helper() + dir := packageDir(t) + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dir, func(fi fs.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + require.NoError(t, err) + + var names []string + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || !fn.Name.IsExported() { + continue + } + if fn.Recv == nil { + if isStringToString(fn.Type) { + names = append(names, fn.Name.Name) + } + continue + } + if receiverTypeName(fn.Recv) == "Redaction" && rendersAValue(fn.Type) { + names = append(names, "Redaction."+fn.Name.Name) + } + } + } + } + sort.Strings(names) + return names +} + +func isStringToString(ft *ast.FuncType) bool { + if ft.Params == nil || ft.Results == nil { + return false + } + if len(ft.Results.List) != 1 || !isStringIdent(ft.Results.List[0].Type) { + return false + } + count := 0 + for _, field := range ft.Params.List { + if !isStringIdent(field.Type) { + return false + } + n := len(field.Names) + if n == 0 { + n = 1 + } + count += n + } + return count == 1 +} + +// receiverTypeName returns the bare type name of a method receiver. +func receiverTypeName(recv *ast.FieldList) string { + if recv == nil || len(recv.List) == 0 { + return "" + } + t := recv.List[0].Type + if star, ok := t.(*ast.StarExpr); ok { + t = star.X + } + if id, ok := t.(*ast.Ident); ok { + return id.Name + } + return "" +} + +// rendersAValue reports whether a method returns exactly one rendered value — +// a `string` or a `[]string`. That is the shape of every mask rendering on +// Redaction, and it excludes the ones that return a policy (WithLimit) or a +// generic tree (Value, RedactNested). +func rendersAValue(ft *ast.FuncType) bool { + if ft.Results == nil || len(ft.Results.List) != 1 { + return false + } + res := ft.Results.List[0].Type + if isStringIdent(res) { + return true + } + arr, ok := res.(*ast.ArrayType) + return ok && arr.Len == nil && isStringIdent(arr.Elt) +} + +func isStringIdent(e ast.Expr) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == "string" +} + +// packageDir returns the directory this test file lives in. +func packageDir(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + require.True(t, ok, "runtime.Caller failed; the source-parsing guards cannot run") + return filepath.Dir(file) +} + +// The defect TestExportedStringFuncs_RenderOnlyRecognisedMasks found on its +// first run, pinned as its own regression. +// +// A URL is re-serialised through net/url after its components are masked, so +// every LIVE read door publishes the mask PERCENT-ENCODED. MaskMarkers carried +// only the decoded spelling, so ContainsMaskMarker — which is +// UnmaskLiveURL's and CheckServerWriteMasks' residual check — did not recognise +// a masked URL at all. A client echoing one back at a write door whose key-bound +// revert could not bind it (host changed, parameter count changed, the mask sits +// in the path) had the MASK persisted over the live credential: the #1142 +// corruption, still open after seven rounds. +func TestEscapedURLMask_IsRefusedOnTheWritePath(t *testing.T) { + const stored = "https://host/mcp?access_token=supersecretvalue" + + masked := LiveRedaction.URLValue(stored) + require.NotEqual(t, stored, masked, "precondition: the read door masks this URL") + require.NotContains(t, masked, "supersecretvalue") + assert.True(t, ContainsMaskMarker(masked), + "the fail-closed net must recognise the URL rendering its own read doors publish: %q", masked) + + // A caller that edits the host cannot have the stored secret restored, so + // the mask must be refused rather than written through. + moved := strings.Replace(masked, "https://host/", "https://attacker.example/", 1) + _, err := UnmaskLiveURL(moved, stored) + require.Error(t, err, "an unbindable masked URL was ACCEPTED and would be persisted over the credential") + + // And the generic residual net, which is what makes a future field fail closed. + require.Error(t, CheckServerWriteMasks("server.", map[string]interface{}{"url": moved})) +} diff --git a/internal/oauth/serverfields.go b/internal/oauth/serverfields.go index bffc29759..3ae14a705 100644 --- a/internal/oauth/serverfields.go +++ b/internal/oauth/serverfields.go @@ -53,6 +53,20 @@ func RedactServerSecretFields(server *contracts.Server) { if server.URL != "" { server.URL = LiveRedaction.URLValue(server.URL) } + // Round 8 finding 3: `command` and `working_dir` are ONE leaf with THREE + // doors, and this one published them RAW while the generic MCP view (the + // shared walk behind `quarantine_security list_quarantined`) ran the leaf + // rule over them. A command path is rarely a credential, which is a reason + // for the RULE to leave it readable — the shared leaf rule passes `npx` and + // `/opt/tools` through byte-identical — not a reason for two doors to + // answer differently about the same string. The rule is also the only thing + // that catches a token pasted into either field. + if server.Command != "" { + server.Command = LiveRedaction.Leaf("command", server.Command) + } + if server.WorkingDir != "" { + server.WorkingDir = LiveRedaction.Leaf("working_dir", server.WorkingDir) + } if len(server.Args) > 0 { server.Args = LiveRedaction.Argv(server.Args) } @@ -126,7 +140,7 @@ func RedactServerSecretFields(server *contracts.Server) { } } if server.LastError != "" { - server.LastError = MaskDetectedSecrets(RedactSensitiveData(server.LastError)) + server.LastError = ScrubUpstreamText(server.LastError) } // GH #1145's retry_stopped_reason is free-form UPSTREAM text, not the fixed // catalog string its name suggests: diagnostics.PermanentFailureReason falls @@ -141,7 +155,7 @@ func RedactServerSecretFields(server *contracts.Server) { } if server.Health != nil && server.Health.Detail != "" { health := *server.Health - health.Detail = MaskDetectedSecrets(RedactSensitiveData(health.Detail)) + health.Detail = ScrubUpstreamText(health.Detail) server.Health = &health } // Spec 044 diagnostic — its Cause echoes the raw connect error, which @@ -149,7 +163,7 @@ func RedactServerSecretFields(server *contracts.Server) { // with LastError / Health.Detail. if server.Diagnostic != nil && server.Diagnostic.Cause != "" { diag := *server.Diagnostic - diag.Cause = MaskDetectedSecrets(RedactSensitiveData(diag.Cause)) + diag.Cause = ScrubUpstreamText(diag.Cause) server.Diagnostic = &diag } } @@ -214,12 +228,21 @@ var ServerFieldMaskDecisions = map[string]MaskDecision{ // --- masked on read, never reverted --- "args": MaskDecisionRefuse, // CheckArgvMaskEcho: an argv slot carries no key + // Round 8 finding 3. These two used to be recorded as not-secret, which was + // the THIRD answer to a leaf two doors already disagreed about: the generic + // MCP view masked them, the REST/SSE door and `upstream_servers list` + // published them raw. Every read door now runs the shared leaf rule over + // them — which leaves an ordinary command or path byte-identical and masks + // a token pasted into either — and nothing binds a revert to a single + // string field the caller replaces wholesale, so an echoed mask is refused + // by the residual net. + "command": MaskDecisionRefuse, + "working_dir": MaskDecisionRefuse, + // --- not secret-bearing --- "id": MaskDecisionNotSecret, "name": MaskDecisionNotSecret, "protocol": MaskDecisionNotSecret, - "command": MaskDecisionNotSecret, - "working_dir": MaskDecisionNotSecret, "enabled": MaskDecisionNotSecret, "quarantined": MaskDecisionNotSecret, "skip_quarantine": MaskDecisionNotSecret, @@ -376,3 +399,50 @@ func init() { ServerFieldMaskDecisions[path] = decision } } + +// RedactedConfigView renders one config.ServerConfig — or any other config +// struct that carries server-derived strings — as a generic JSON map with every +// secret-bearing leaf masked under the LIVE policy. +// +// Issue #1148, round 8. internal/server has had this shape since round 1 +// (`redactedServerView`), but it lived up there, so the doors BELOW it in the +// import graph could not reach it: the server edition's `/admin/servers` +// fallback wrote `h.config.Servers` — the raw []*config.ServerConfig, every env +// value, header and oauth.client_secret in it — straight to JSON, and its +// shared-toggle handler echoed back a raw *config.ServerConfig. The view moves +// here so EVERY door can build a payload from a redacted view rather than from +// the struct. +// +// `key` is the enclosing wire field name; pass "" for a whole server config. +// Returns nil for nil. A value that cannot round-trip through generic JSON +// fails CLOSED: nil is returned rather than the raw struct. +func RedactedConfigView(key string, v interface{}) map[string]interface{} { + if v == nil { + return nil + } + normalized, ok := NormalizeForRedaction(v).(map[string]interface{}) + if !ok { + return nil + } + redacted, ok := LiveRedaction.Value(key, normalized).(map[string]interface{}) + if !ok { + return nil + } + return redacted +} + +// RedactedConfigViews is RedactedConfigView over a slice, skipping nils. +// Returns an empty (non-nil) slice for an empty input so JSON callers keep +// emitting `[]`. +func RedactedConfigViews[T any](key string, items []*T) []map[string]interface{} { + out := make([]map[string]interface{}, 0, len(items)) + for _, item := range items { + if item == nil { + continue + } + if view := RedactedConfigView(key, item); view != nil { + out = append(out, view) + } + } + return out +} diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 3ca45f55e..f0d3b5fa7 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -2022,8 +2022,11 @@ func (r *Runtime) ListRegistries() ([]interface{}, error) { "id": reg.ID, "name": reg.Name, "description": reg.Description, - "url": reg.URL, - "servers_url": reg.ServersURL, + // Issue #1148, round 8: a custom registry source is + // operator-configured, so its URL can carry a query credential + // exactly as an upstream URL can. Same shared rule. + "url": oauth.LiveRedaction.URLValue(reg.URL), + "servers_url": oauth.LiveRedaction.URLValue(reg.ServersURL), "tags": reg.Tags, "protocol": reg.Protocol, "count": reg.Count, diff --git a/internal/server/add_registry_source.go b/internal/server/add_registry_source.go index 5ef903d0f..57b206dbf 100644 --- a/internal/server/add_registry_source.go +++ b/internal/server/add_registry_source.go @@ -12,6 +12,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/registries" ) @@ -90,6 +91,16 @@ func (s *Server) AddRegistrySource(req *AddRegistrySourceRequest) (*config.Regis if req == nil { return nil, errors.New("nil request") } + // Issue #1148, round 8: the read doors mask a registry source URL with the + // shared LIVE rule, so a client can echo that mask back here. Nothing binds + // a revert to a registry URL — the caller replaces it wholesale — so the + // answer is the same bind-or-refuse one the server write path gives: + // refuse, and let the caller resend the real value. + if err := oauth.CheckServerWriteMasks("registry_source.", map[string]interface{}{ + "url": req.URL, "servers_url": "", + }); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRegistryURL, err) + } entry, err := buildRegistrySourceEntry(req.URL, req.Protocol, req.ID, req.Name) if err != nil { diff --git a/internal/server/edit_registry_source.go b/internal/server/edit_registry_source.go index e11110b78..41b720d19 100644 --- a/internal/server/edit_registry_source.go +++ b/internal/server/edit_registry_source.go @@ -10,6 +10,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/registries" ) @@ -57,6 +58,15 @@ func (s *Server) EditRegistrySource(req *EditRegistrySourceRequest) (*config.Reg return nil, errors.New("nil request") } + // Issue #1148, round 8: an echoed read mask is REFUSED, never persisted + // over the stored credential — the same answer the server write path gives + // for every field with no key-bound revert. + if err := oauth.CheckServerWriteMasks("registry_source.", map[string]interface{}{ + "url": req.URL, "servers_url": req.ServersURL, + }); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRegistryURL, err) + } + currentConfig := s.runtime.Config() if currentConfig == nil { return nil, errors.New("configuration unavailable") diff --git a/internal/server/mcp.go b/internal/server/mcp.go index 93160802a..cce2fc26e 100644 --- a/internal/server/mcp.go +++ b/internal/server/mcp.go @@ -1524,9 +1524,10 @@ func (p *MCPProxyServer) handleListRegistries(ctx context.Context, _ mcp.CallToo "id": reg.ID, "name": reg.Name, "description": reg.Description, - "url": reg.URL, - "tags": reg.Tags, - "count": reg.Count, + // Issue #1148, round 8: same shared rule as every other URL door. + "url": liveRedaction.URLValue(reg.URL), + "tags": reg.Tags, + "count": reg.Count, // MCP-866: provenance/trust so an agent can tell official from // user-added third-party sources. "provenance": reg.Provenance, @@ -3267,7 +3268,7 @@ func (p *MCPProxyServer) handleAddServerFromRegistry(ctx context.Context, reques summary := contracts.AddedServerSummary{ Name: cfg.Name, Protocol: cfg.Protocol, - Command: cfg.Command, + Command: viewString(registryView, "command", cfg.Command), Args: redactedArgs(cfg.Args, liveRedaction), URL: viewString(registryView, "url", cfg.URL), Enabled: cfg.Enabled, @@ -3792,10 +3793,17 @@ func (p *MCPProxyServer) handleListUpstreams(ctx context.Context) (*mcp.CallTool serverURL = viewField(view, "url", server.URL) serverArgs = redactedArgs(server.Args, liveRedaction) } + serverCommand := interface{}(server.Command) + if !revealHeaders { + // Round 8 finding 3: `command` is the one leaf of this projection + // that was still read off the struct instead of the view, so it + // was masked on `list_quarantined` and raw here. + serverCommand = viewField(view, "command", server.Command) + } serverMap := map[string]interface{}{ "name": server.Name, "protocol": server.Protocol, - "command": server.Command, + "command": serverCommand, "args": serverArgs, "url": serverURL, "env": serverEnv, @@ -5763,12 +5771,16 @@ func (p *MCPProxyServer) createDetailedErrorResponse(err error, serverName, tool // Check if it's our enhanced error types if errors.As(err, &httpErr) { // We have HTTP error details + // Issue #1148, round 8: `server_url` is the FULL upstream request URL, + // query credentials and all, and `response_body` is whatever the + // upstream printed — both went out of the MCP surface verbatim. Route + // them through the same two shared rules every other door uses. errorDetails := map[string]interface{}{ - "error": httpErr.Error(), + "error": scrubUpstreamText(httpErr.Error()), "http_details": map[string]interface{}{ "status_code": httpErr.StatusCode, - "response_body": httpErr.Body, - "server_url": httpErr.URL, + "response_body": scrubUpstreamText(httpErr.Body), + "server_url": liveRedaction.URLValue(httpErr.URL), "method": httpErr.Method, }, "troubleshooting": p.generateTroubleshootingAdvice(httpErr.StatusCode, httpErr.Body), @@ -5781,16 +5793,16 @@ func (p *MCPProxyServer) createDetailedErrorResponse(err error, serverName, tool if errors.As(err, &jsonRPCErr) { // We have JSON-RPC error details errorDetails := map[string]interface{}{ - "error": jsonRPCErr.Message, + "error": scrubUpstreamText(jsonRPCErr.Message), "error_code": jsonRPCErr.Code, - "error_data": jsonRPCErr.Data, + "error_data": redactValueWith("error_data", normalizeForRedaction(jsonRPCErr.Data), liveRedaction), } if jsonRPCErr.HTTPError != nil { errorDetails["http_details"] = map[string]interface{}{ "status_code": jsonRPCErr.HTTPError.StatusCode, - "response_body": jsonRPCErr.HTTPError.Body, - "server_url": jsonRPCErr.HTTPError.URL, + "response_body": scrubUpstreamText(jsonRPCErr.HTTPError.Body), + "server_url": liveRedaction.URLValue(jsonRPCErr.HTTPError.URL), } errorDetails["troubleshooting"] = p.generateTroubleshootingAdvice(jsonRPCErr.HTTPError.StatusCode, jsonRPCErr.HTTPError.Body) } diff --git a/internal/server/mcp_server_view.go b/internal/server/mcp_server_view.go index e8336abdd..c4d0fc4b6 100644 --- a/internal/server/mcp_server_view.go +++ b/internal/server/mcp_server_view.go @@ -78,10 +78,7 @@ func redactedServerViews(servers []*config.ServerConfig, r redactionPolicy) []ma // debugging surface and a long line is precisely what an operator opens it for. // The cap now lives on scrubUpstreamTextForAudit, where it belongs. func scrubUpstreamText(s string) string { - if s == "" { - return s - } - return maskDetectedSecrets(oauth.RedactSensitiveData(s)) + return oauth.ScrubUpstreamText(s) } // scrubUpstreamTextForAudit is scrubUpstreamText for a string that will be diff --git a/internal/server/server.go b/internal/server/server.go index 49c5b4aa5..beb8e465a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -29,6 +29,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/httpapi" "github.com/smart-mcp-proxy/mcpproxy-go/internal/logs" "github.com/smart-mcp-proxy/mcpproxy-go/internal/management" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/observability" "github.com/smart-mcp-proxy/mcpproxy-go/internal/profile" "github.com/smart-mcp-proxy/mcpproxy-go/internal/runtime" @@ -1193,6 +1194,17 @@ func (s *Server) SuggestAlternateListen(baseAddr string) (string, error) { // GetUpstreamStats returns statistics about upstream servers func (s *Server) GetUpstreamStats() map[string]interface{} { + // Issue #1148, round 8. This is the SECOND implementation of + // `upstream_stats` and the one that actually runs: round 4 masked + // upstream.Manager.GetStats, which is only the fallback below, so the + // StateView path served `url` and `last_error` verbatim on + // GET /api/v1/status and on every SSE status event. Both implementations + // now hand their map to the one shared rule at this boundary. + reveal := false + if cfg, err := s.GetConfig(); err == nil && cfg != nil { + reveal = cfg.RevealSecretHeaders + } + if supervisor := s.runtime.Supervisor(); supervisor != nil { if view := supervisor.StateView(); view != nil { snapshot := view.Snapshot() @@ -1305,17 +1317,20 @@ func (s *Server) GetUpstreamStats() map[string]interface{} { serverStats[name] = entry } - return map[string]interface{}{ + return oauth.RedactUpstreamStats(map[string]interface{}{ "connected_servers": connectedCount, "connecting_servers": connectingCount, "quarantined_servers": quarantinedCount, "total_servers": len(snapshot.Servers), "servers": serverStats, "total_tools": totalTools, - } + }, reveal) } } + // The fallback masks its own entries (upstream.Manager.GetStats applies the + // same oauth.RedactUpstreamStatsEntry rule with the same reveal gate), so + // this path is already redacted by the time it gets here. stats := s.runtime.UpstreamManager().GetStats() // Enhance stats with tool counts per server when falling back diff --git a/internal/serveredition/api/admin_handlers.go b/internal/serveredition/api/admin_handlers.go index 39d03e5dd..1d7bb1e4f 100644 --- a/internal/serveredition/api/admin_handlers.go +++ b/internal/serveredition/api/admin_handlers.go @@ -16,6 +16,7 @@ import ( "github.com/smart-mcp-proxy/mcpproxy-go/internal/auth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" "github.com/smart-mcp-proxy/mcpproxy-go/internal/contracts" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" teamsauth "github.com/smart-mcp-proxy/mcpproxy-go/internal/serveredition/auth" "github.com/smart-mcp-proxy/mcpproxy-go/internal/serveredition/multiuser" "github.com/smart-mcp-proxy/mcpproxy-go/internal/serveredition/users" @@ -481,9 +482,15 @@ func (h *AdminHandlers) toggleSharedServer(w http.ResponseWriter, r *http.Reques } h.logger.Infow("server shared status toggled", "server", name, "shared", req.Shared) + // Issue #1148, round 8: `found` is a raw *config.ServerConfig — every env + // value, header, oauth.client_secret and URL credential the operator + // configured. Echoing it back handed the whole upstream credential set to + // any admin session over the server edition's REST API. Build the echo from + // the shared redacted view instead, exactly as every personal-edition door + // does. writeJSON(w, http.StatusOK, map[string]interface{}{ "message": fmt.Sprintf("Server %q shared status set to %v", name, req.Shared), - "server": found, + "server": oauth.RedactedConfigView("", found), }) } @@ -499,8 +506,11 @@ func (h *AdminHandlers) listAdminServers(w http.ResponseWriter, r *http.Request) } mgmt, ok := h.managementSvc.(lister) if !ok { - // Fallback to config-only listing when management service is not available. - writeJSON(w, http.StatusOK, h.config.Servers) + // Issue #1148, round 8: this fallback wrote the raw + // []*config.ServerConfig straight to JSON — the exact shape + // `quarantine_security list_quarantined` was fixed for in round 1, + // still open on the server edition's admin API. One shared view. + writeJSON(w, http.StatusOK, oauth.RedactedConfigViews("", h.config.Servers)) return } @@ -508,7 +518,7 @@ func (h *AdminHandlers) listAdminServers(w http.ResponseWriter, r *http.Request) if err != nil { h.logger.Errorw("failed to list servers via management service", "error", err) // Fallback to config-only listing on error. - writeJSON(w, http.StatusOK, h.config.Servers) + writeJSON(w, http.StatusOK, oauth.RedactedConfigViews("", h.config.Servers)) return } @@ -523,9 +533,18 @@ func (h *AdminHandlers) listAdminServers(w http.ResponseWriter, r *http.Request) *contracts.Server Shared bool `json:"shared"` } + // Issue #1148, round 8: management.ListServers hands back RAW + // contracts.Server values — the personal edition redacts them at its own + // serialization boundary (httpapi.redactServerSecrets), and this door, + // which is the server edition's boundary onto the same structs, did not. + // The copy matters: the projection may still share maps/slices with the + // stored config, and RedactServerSecretFields replaces rather than mutates + // them. enriched := make([]*enrichedServer, len(servers)) for i, s := range servers { - enriched[i] = &enrichedServer{Server: s, Shared: sharedMap[s.Name]} + redacted := *s + oauth.RedactServerSecretFields(&redacted) + enriched[i] = &enrichedServer{Server: &redacted, Shared: sharedMap[s.Name]} } writeJSON(w, http.StatusOK, map[string]interface{}{ diff --git a/internal/upstream/core/monitoring.go b/internal/upstream/core/monitoring.go index ffeb2eb8a..3aa69520c 100644 --- a/internal/upstream/core/monitoring.go +++ b/internal/upstream/core/monitoring.go @@ -11,6 +11,8 @@ import ( "time" "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" ) const ( @@ -449,12 +451,16 @@ func (c *Client) GetConnectionDiagnostics() map[string]interface{} { c.mu.RLock() defer c.mu.RUnlock() + // Issue #1148, round 8: this map is a server-payload shape (command + argv) + // and nothing about it is private to this package, so it takes the shared + // LIVE rule like every other door — before it acquires a caller rather than + // after. diagnostics := map[string]interface{}{ "connected": c.connected, "transport_type": c.transportType, "server_name": c.config.Name, - "command": c.config.Command, - "args": c.config.Args, + "command": oauth.LiveRedaction.Leaf("command", c.config.Command), + "args": oauth.LiveRedaction.Argv(c.config.Args), "has_stderr": c.stderr != nil, "has_process_cmd": c.processCmd != nil, } @@ -470,7 +476,7 @@ func (c *Client) GetConnectionDiagnostics() map[string]interface{} { // Add Docker-specific diagnostics if c.isDockerCommand { diagnostics["is_docker"] = true - diagnostics["docker_args"] = c.config.Args + diagnostics["docker_args"] = oauth.LiveRedaction.Argv(c.config.Args) diagnostics["container_id"] = c.containerID // Check Docker daemon connectivity diff --git a/internal/upstream/manager.go b/internal/upstream/manager.go index e9ece0d04..86cda2ae7 100644 --- a/internal/upstream/manager.go +++ b/internal/upstream/manager.go @@ -1537,7 +1537,7 @@ func (m *Manager) ConnectAll(ctx context.Context) error { m.logger.Info("Attempting to connect client", zap.String("id", id), zap.String("name", client.GetConfig().Name), - zap.String("url", oauth.RedactURLQueryParams(client.GetConfig().URL)), + zap.String("url", oauth.LiveRedaction.URLValue(client.GetConfig().URL)), zap.String("command", client.GetConfig().Command), zap.String("protocol", client.GetConfig().Protocol)) @@ -1739,10 +1739,6 @@ func (m *Manager) GetStats() map[string]interface{} { } } - if !revealSecrets { - url = oauth.RedactURLQueryParams(url) - } - status := map[string]interface{}{ "state": connectionInfo.State.String(), "connected": connectionInfo.State == types.StateReady, @@ -1767,14 +1763,9 @@ func (m *Manager) GetStats() map[string]interface{} { } if connectionInfo.LastError != nil { - lastError := connectionInfo.LastError.Error() - if !revealSecrets { - // A transport error quotes the request URL inside its own - // message (`Post "https://host/mcp?token=…": dial tcp …`), so - // masking the `url` key alone would not close this. - lastError = oauth.RedactSensitiveData(lastError) - } - status["last_error"] = lastError + // Masked below by oauth.RedactUpstreamStats, at the one wire + // boundary both upstream_stats implementations pass through. + status["last_error"] = connectionInfo.LastError.Error() } if connectionInfo.ServerName != "" { @@ -1794,6 +1785,14 @@ func (m *Manager) GetStats() map[string]interface{} { } } + // Round 8 finding 1: `url` and `last_error` are masked by the ONE + // shared rule (oauth.RedactUpstreamStatsEntry), not by name-only + // redactors chosen here — the server list in the SAME response applies + // exactly that rule to exactly these two strings. + if !revealSecrets { + oauth.RedactUpstreamStatsEntry(status) + } + serverStatus[id] = status } diff --git a/internal/upstream/round8_stats_rule_test.go b/internal/upstream/round8_stats_rule_test.go new file mode 100644 index 000000000..6f58e5a07 --- /dev/null +++ b/internal/upstream/round8_stats_rule_test.go @@ -0,0 +1,100 @@ +package upstream + +import ( + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/config" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/oauth" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/storage" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/upstream/managed" +) + +// Issue #1148, round 8 finding 1. +// +// GetStats builds `upstream_stats`, which is served verbatim on +// GET /api/v1/status, on GET /api/v1/servers and on the initial SSE `status` +// event — the SAME responses that carry the redacted server list. It masked its +// `url` with the NAME-rule-only oauth.RedactURLQueryParams and its `last_error` +// with the NAME-rule-only oauth.RedactSensitiveData, while the sibling list in +// the same response ran the name rule PLUS the value-shaped detector. So a +// credential under a query parameter no matcher recognises was masked in one +// half of a single response and published in the clear in the other. + +const round8GHPToken = "ghp_1234567890abcdefghijABCDEFGHIJ123456" + +func newStatsManager(t *testing.T, serverConfig *config.ServerConfig, global *config.Config) (*Manager, *managed.Client) { + t.Helper() + logger := zap.NewNop() + db, err := storage.NewBoltDB(t.TempDir(), logger.Sugar()) + require.NoError(t, err) + t.Cleanup(func() { db.Close() }) + + manager := &Manager{ + clients: make(map[string]*managed.Client), + logger: logger, + storage: db, + secretResolver: secret2Resolver(), + } + manager.globalConfig.Store(global) + client, err := managed.NewClient(serverConfig.Name, serverConfig, logger, nil, &config.Config{}, db, secret2Resolver()) + require.NoError(t, err) + manager.clients[serverConfig.Name] = client + return manager, client +} + +func TestGetStats_UsesTheSharedURLRuleAsTheServerList(t *testing.T) { + rawURL := "https://h/mcp?opaque=" + round8GHPToken + manager, _ := newStatsManager(t, &config.ServerConfig{ + Name: "leaky", Protocol: "http", URL: rawURL, Enabled: true, + }, &config.Config{}) + + rendered := fmt.Sprint(manager.GetStats()) + assert.NotContains(t, rendered, round8GHPToken, + "upstream_stats published a credential the sibling server list in the SAME response masks") + assert.Equal(t, oauth.LiveRedaction.URLValue(rawURL), statsServerField(t, manager, "leaky", "url"), + "upstream_stats must render `url` with the one shared live rule") +} + +func TestGetStats_UsesTheSharedFreeTextRuleForLastError(t *testing.T) { + msg := `Post "https://h/mcp?opaque=` + round8GHPToken + `": dial tcp: refused` + manager, client := newStatsManager(t, &config.ServerConfig{ + Name: "leaky", Protocol: "http", URL: "https://h/mcp", Enabled: true, + }, &config.Config{}) + client.StateManager.SetError(errors.New(msg)) + + rendered := fmt.Sprint(manager.GetStats()) + assert.NotContains(t, rendered, round8GHPToken, + "upstream_stats published a credential carried by last_error") + assert.Equal(t, oauth.ScrubUpstreamText(msg), statsServerField(t, manager, "leaky", "last_error"), + "upstream_stats must scrub `last_error` with the one shared free-text rule") +} + +func TestGetStats_RevealSecretHeadersStillOptsOut(t *testing.T) { + rawURL := "https://h/mcp?opaque=" + round8GHPToken + manager, client := newStatsManager(t, &config.ServerConfig{ + Name: "leaky", Protocol: "http", URL: rawURL, Enabled: true, + }, &config.Config{RevealSecretHeaders: true}) + client.StateManager.SetError(errors.New("boom " + round8GHPToken)) + + rendered := fmt.Sprint(manager.GetStats()) + assert.Contains(t, rendered, rawURL, "the opt-out must still expose the real URL") + assert.Contains(t, rendered, "boom "+round8GHPToken, "the opt-out must still expose the real error") +} + +func statsServerField(t *testing.T, m *Manager, server, field string) string { + t.Helper() + stats := m.GetStats() + servers, ok := stats["servers"].(map[string]interface{}) + require.True(t, ok, "upstream_stats must carry a servers map: %#v", stats) + entry, ok := servers[server].(map[string]interface{}) + require.True(t, ok, "no entry for %q: %#v", server, servers) + value, ok := entry[field].(string) + require.True(t, ok, "%q is not a string: %#v", field, entry) + return value +}