From 150b2c3f283788af16ee326e12301f75e6eff6ce Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:25:56 -0400 Subject: [PATCH] fix: keep the last raw-byte truncations on a rune boundary Round 3 of the UTF-8-safe truncation sweep (#1647, #1664, #1665). Three production sites still cut text with a raw byte slice, so a non-ASCII value whose cut offset lands inside a multi-byte rune is published as invalid UTF-8 and JSON encoding rewrites the tail to U+FFFD. - internal/storage: truncateResponse caps the activity-log response at activity_max_response_size. It now goes through truncateRuneSafe, the helper the per-server tool-call records in the same package already use, so the stored copy, GET /api/v1/activities and the CSV export all stay valid UTF-8. - internal/outputvalidation: the 500-byte schema-violation Reason. A jsonschema message quotes the offending instance location, so a non-ASCII property name or value sits astride the cut. - internal/registries: the 300-byte Pulse description cap. Registry descriptions are upstream-authored free text. ASCII behaviour is unchanged at all three sites; the cut never exceeds the cap. --- internal/outputvalidation/reason_utf8_test.go | 45 +++++++++++++++++++ internal/outputvalidation/validator.go | 11 ++++- internal/registries/description_utf8_test.go | 41 +++++++++++++++++ internal/registries/search.go | 23 +++++----- internal/storage/activity.go | 8 +++- .../storage/activity_utf8_truncation_test.go | 45 +++++++++++++++++++ 6 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 internal/outputvalidation/reason_utf8_test.go create mode 100644 internal/registries/description_utf8_test.go create mode 100644 internal/storage/activity_utf8_truncation_test.go diff --git a/internal/outputvalidation/reason_utf8_test.go b/internal/outputvalidation/reason_utf8_test.go new file mode 100644 index 000000000..35dfeb21b --- /dev/null +++ b/internal/outputvalidation/reason_utf8_test.go @@ -0,0 +1,45 @@ +package outputvalidation + +import ( + "fmt" + "strings" + "testing" + "unicode/utf8" +) + +// A schema-violation Reason is capped at 500 bytes, and jsonschema's message +// quotes the offending instance location — so a non-ASCII property name or +// value puts multi-byte runes astride the cut. The raw byte slice left the +// Reason invalid UTF-8, which JSON encoding rewrites to U+FFFD on the way to +// the agent and the audit line (round 3 of #1647/#1664/#1665). +// +// The 1-byte padding shifts every later byte, so the loop covers all three +// offsets inside a 3-byte rune; two of the three are mid-rune. +func TestViolationReasonIsValidUTF8(t *testing.T) { + v := New(0, 0, nil) + for pad := 0; pad < 3; pad++ { + name := strings.Repeat("a", pad) + strings.Repeat("語", 200) + schema := fmt.Sprintf(`{"type":"object","properties":{%q:{"type":"integer"}}}`, name) + + verdict := v.Validate(fmt.Sprintf("srv:tool%d", pad), schema, map[string]any{name: "not-an-integer"}) + if !verdict.IsViolation() { + t.Fatalf("pad=%d: expected a violation", pad) + } + if len(verdict.Reason) > 500 { + t.Errorf("pad=%d: reason is %d bytes, over the 500-byte cap", pad, len(verdict.Reason)) + } + if !utf8.ValidString(verdict.Reason) { + t.Errorf("pad=%d: reason is not valid UTF-8 (cut inside a rune)", pad) + } + } +} + +// ASCII behaviour must not move. +func TestTruncateASCIIUnchanged(t *testing.T) { + if got, want := truncate(strings.Repeat("x", 10), 20), strings.Repeat("x", 10); got != want { + t.Errorf("under the cap: got %q, want %q", got, want) + } + if got, want := truncate(strings.Repeat("x", 30), 20), strings.Repeat("x", 17)+"..."; got != want { + t.Errorf("over the cap: got %q, want %q", got, want) + } +} diff --git a/internal/outputvalidation/validator.go b/internal/outputvalidation/validator.go index 3fd070ae9..c59e908c2 100644 --- a/internal/outputvalidation/validator.go +++ b/internal/outputvalidation/validator.go @@ -11,6 +11,8 @@ import ( "github.com/santhosh-tekuri/jsonschema/v6" "go.uber.org/zap" + + "github.com/smart-mcp-proxy/mcpproxy-go/internal/stringutil" ) // compileCount counts how many times a schema is actually compiled (vs served @@ -225,10 +227,15 @@ func hashSchema(schemaJSON string) uint64 { return h.Sum64() } -// truncate shortens s to at most maxLen characters, appending "..." if truncated. +// truncate shortens s to at most maxLen bytes, appending "..." if truncated. +// +// The cut goes through stringutil.UTF8Prefix: a jsonschema message quotes the +// offending instance location, so a non-ASCII property name or value puts +// multi-byte runes astride the cut, and a raw byte slice leaves the Reason +// invalid UTF-8 (U+FFFD once JSON-encoded for the agent and the audit line). func truncate(s string, maxLen int) string { if len(s) <= maxLen { return s } - return s[:maxLen-3] + "..." + return stringutil.UTF8Prefix(s, maxLen-3) + "..." } diff --git a/internal/registries/description_utf8_test.go b/internal/registries/description_utf8_test.go new file mode 100644 index 000000000..18090c033 --- /dev/null +++ b/internal/registries/description_utf8_test.go @@ -0,0 +1,41 @@ +package registries + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "unicode/utf8" +) + +// PulseMCP descriptions are upstream-authored free text, so a non-Latin +// description longer than the 300-byte display cap put multi-byte runes +// astride the cut. The raw byte slice left ServerEntry.Description invalid +// UTF-8, which the REST/Web UI JSON encoder rewrites to U+FFFD (round 3 of +// #1647/#1664/#1665). +// +// The 1-byte padding covers all three offsets inside a 3-byte rune. +func TestPulseDescriptionTruncationIsValidUTF8(t *testing.T) { + for _, field := range []string{"short_description", "EXPERIMENTAL_ai_generated_description"} { + for pad := 0; pad < 3; pad++ { + desc := strings.Repeat("a", pad) + strings.Repeat("語", 200) + payload := fmt.Sprintf(`{"servers":[{"name":"srv",%q:%q}]}`, field, desc) + + var raw map[string]interface{} + if err := json.Unmarshal([]byte(payload), &raw); err != nil { + t.Fatalf("fixture is not JSON: %v", err) + } + entries := parsePulseWithoutGuesser(raw) + if len(entries) != 1 { + t.Fatalf("%s pad=%d: parsed %d entries, want 1", field, pad, len(entries)) + } + got := entries[0].Description + if len(got) > 300 { + t.Errorf("%s pad=%d: description is %d bytes, over the 300-byte cap", field, pad, len(got)) + } + if !utf8.ValidString(got) { + t.Errorf("%s pad=%d: description is not valid UTF-8 (cut inside a rune)", field, pad) + } + } + } +} diff --git a/internal/registries/search.go b/internal/registries/search.go index a01dc4ba6..2ad515ee4 100644 --- a/internal/registries/search.go +++ b/internal/registries/search.go @@ -9,8 +9,13 @@ import ( "strings" "github.com/smart-mcp-proxy/mcpproxy-go/internal/experiments" + "github.com/smart-mcp-proxy/mcpproxy-go/internal/stringutil" ) +// pulseDescriptionMaxBytes caps the description carried on a Pulse registry +// search result. +const pulseDescriptionMaxBytes = 300 + // Sentinel errors for single-server lookup. Surfaces map these to stable error // codes (registry_not_found / server_not_found) for cross-surface consistency // (CN-004). @@ -702,19 +707,15 @@ func parsePulseWithoutGuesser(rawData interface{}) []ServerEntry { Name: name, } - // Try to get description from multiple fields + // Try to get description from multiple fields. The display cap goes + // through stringutil.UTF8Prefix: a registry description is + // upstream-authored free text, and a raw byte cut at 300 can land + // inside a multi-byte rune, leaving the entry invalid UTF-8 (U+FFFD + // once the REST/Web UI encoder sees it). if shortDesc, ok := itemMap["short_description"].(string); ok && shortDesc != "" { - if len(shortDesc) > 300 { - server.Description = shortDesc[:300] - } else { - server.Description = shortDesc - } + server.Description = stringutil.UTF8Prefix(shortDesc, pulseDescriptionMaxBytes) } else if aiDesc, ok := itemMap["EXPERIMENTAL_ai_generated_description"].(string); ok && aiDesc != "" { - if len(aiDesc) > 300 { - server.Description = aiDesc[:300] - } else { - server.Description = aiDesc - } + server.Description = stringutil.UTF8Prefix(aiDesc, pulseDescriptionMaxBytes) } else { server.Description = noDescAvailable } diff --git a/internal/storage/activity.go b/internal/storage/activity.go index 64bf5b394..4f24aea13 100644 --- a/internal/storage/activity.go +++ b/internal/storage/activity.go @@ -32,6 +32,12 @@ func parseActivityKey(key []byte) string { // truncateResponse truncates a response string if it exceeds maxSize. // Returns the (potentially truncated) string and whether truncation occurred. +// +// The cut goes through truncateRuneSafe, the same in-package helper the +// per-server tool-call records use: an upstream response is arbitrary UTF-8, +// and a raw byte cut at maxSize can land inside a multi-byte rune. The record +// is then persisted and served over /api/v1/activities with invalid UTF-8, +// which JSON encoding rewrites to U+FFFD. func truncateResponse(response string, maxSize int) (string, bool) { if maxSize <= 0 { maxSize = DefaultMaxResponseSize @@ -39,7 +45,7 @@ func truncateResponse(response string, maxSize int) (string, bool) { if len(response) <= maxSize { return response, false } - return response[:maxSize] + "...[truncated]", true + return truncateRuneSafe(response, maxSize) + truncationMarker, true } // SaveActivity stores an activity record in BBolt. diff --git a/internal/storage/activity_utf8_truncation_test.go b/internal/storage/activity_utf8_truncation_test.go new file mode 100644 index 000000000..abfa4f739 --- /dev/null +++ b/internal/storage/activity_utf8_truncation_test.go @@ -0,0 +1,45 @@ +package storage + +import ( + "strings" + "testing" + "unicode/utf8" +) + +// Round 3 of the UTF-8 truncation sweep (#1647, #1664, #1665): the activity +// log's own per-record cut was still a raw byte slice, so a response whose +// cut offset lands inside a multi-byte rune was persisted — and served over +// /api/v1/activities and the CSV export — as invalid UTF-8, which JSON +// encoding rewrites to U+FFFD. +// +// Every offset inside a 3-byte rune is exercised, because only two of the +// three are mid-rune: a single fixed cap would pass by luck. +func TestTruncateResponseKeepsRuneBoundary(t *testing.T) { + body := strings.Repeat("語", 200) // 3 bytes per rune + + for _, maxSize := range []int{100, 101, 102, 103, 104, 105} { + got, truncated := truncateResponse(body, maxSize) + if !truncated { + t.Fatalf("maxSize=%d: expected truncation of a %d-byte body", maxSize, len(body)) + } + kept := strings.TrimSuffix(got, "...[truncated]") + if !utf8.ValidString(kept) { + t.Errorf("maxSize=%d: stored response ends in a partial rune: % x", + maxSize, []byte(kept)[max(0, len(kept)-4):]) + } + if len(kept) > maxSize { + t.Errorf("maxSize=%d: kept %d bytes, which is over the cap", maxSize, len(kept)) + } + } +} + +// ASCII behaviour must not move: the cap is still applied exactly. +func TestTruncateResponseASCIIUnchanged(t *testing.T) { + got, truncated := truncateResponse(strings.Repeat("x", 500), 100) + if !truncated { + t.Fatal("expected truncation") + } + if want := strings.Repeat("x", 100) + "...[truncated]"; got != want { + t.Errorf("ASCII cut moved:\n got %q\nwant %q", got, want) + } +}