Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions internal/outputvalidation/reason_utf8_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package outputvalidation

import (
"fmt"
"strings"
"testing"
"unicode/utf8"
)

// A schema-violation Reason is capped at 500 bytes, and jsonschema's message
// quotes the offending instance location — so a non-ASCII property name or
// value puts multi-byte runes astride the cut. The raw byte slice left the
// Reason invalid UTF-8, which JSON encoding rewrites to U+FFFD on the way to
// the agent and the audit line (round 3 of #1647/#1664/#1665).
//
// The 1-byte padding shifts every later byte, so the loop covers all three
// offsets inside a 3-byte rune; two of the three are mid-rune.
func TestViolationReasonIsValidUTF8(t *testing.T) {
v := New(0, 0, nil)
for pad := 0; pad < 3; pad++ {
name := strings.Repeat("a", pad) + strings.Repeat("語", 200)
schema := fmt.Sprintf(`{"type":"object","properties":{%q:{"type":"integer"}}}`, name)

verdict := v.Validate(fmt.Sprintf("srv:tool%d", pad), schema, map[string]any{name: "not-an-integer"})
if !verdict.IsViolation() {
t.Fatalf("pad=%d: expected a violation", pad)
}
if len(verdict.Reason) > 500 {
t.Errorf("pad=%d: reason is %d bytes, over the 500-byte cap", pad, len(verdict.Reason))
}
if !utf8.ValidString(verdict.Reason) {
t.Errorf("pad=%d: reason is not valid UTF-8 (cut inside a rune)", pad)
}
}
}

// ASCII behaviour must not move.
func TestTruncateASCIIUnchanged(t *testing.T) {
if got, want := truncate(strings.Repeat("x", 10), 20), strings.Repeat("x", 10); got != want {
t.Errorf("under the cap: got %q, want %q", got, want)
}
if got, want := truncate(strings.Repeat("x", 30), 20), strings.Repeat("x", 17)+"..."; got != want {
t.Errorf("over the cap: got %q, want %q", got, want)
}
}
11 changes: 9 additions & 2 deletions internal/outputvalidation/validator.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (

"github.com/santhosh-tekuri/jsonschema/v6"
"go.uber.org/zap"

"github.com/smart-mcp-proxy/mcpproxy-go/internal/stringutil"
)

// compileCount counts how many times a schema is actually compiled (vs served
Expand Down Expand Up @@ -225,10 +227,15 @@ func hashSchema(schemaJSON string) uint64 {
return h.Sum64()
}

// truncate shortens s to at most maxLen characters, appending "..." if truncated.
// truncate shortens s to at most maxLen bytes, appending "..." if truncated.
//
// The cut goes through stringutil.UTF8Prefix: a jsonschema message quotes the
// offending instance location, so a non-ASCII property name or value puts
// multi-byte runes astride the cut, and a raw byte slice leaves the Reason
// invalid UTF-8 (U+FFFD once JSON-encoded for the agent and the audit line).
func truncate(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen-3] + "..."
return stringutil.UTF8Prefix(s, maxLen-3) + "..."
}
41 changes: 41 additions & 0 deletions internal/registries/description_utf8_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package registries

import (
"encoding/json"
"fmt"
"strings"
"testing"
"unicode/utf8"
)

// PulseMCP descriptions are upstream-authored free text, so a non-Latin
// description longer than the 300-byte display cap put multi-byte runes
// astride the cut. The raw byte slice left ServerEntry.Description invalid
// UTF-8, which the REST/Web UI JSON encoder rewrites to U+FFFD (round 3 of
// #1647/#1664/#1665).
//
// The 1-byte padding covers all three offsets inside a 3-byte rune.
func TestPulseDescriptionTruncationIsValidUTF8(t *testing.T) {
for _, field := range []string{"short_description", "EXPERIMENTAL_ai_generated_description"} {
for pad := 0; pad < 3; pad++ {
desc := strings.Repeat("a", pad) + strings.Repeat("語", 200)
payload := fmt.Sprintf(`{"servers":[{"name":"srv",%q:%q}]}`, field, desc)

var raw map[string]interface{}
if err := json.Unmarshal([]byte(payload), &raw); err != nil {
t.Fatalf("fixture is not JSON: %v", err)
}
entries := parsePulseWithoutGuesser(raw)
if len(entries) != 1 {
t.Fatalf("%s pad=%d: parsed %d entries, want 1", field, pad, len(entries))
}
got := entries[0].Description
if len(got) > 300 {
t.Errorf("%s pad=%d: description is %d bytes, over the 300-byte cap", field, pad, len(got))
}
if !utf8.ValidString(got) {
t.Errorf("%s pad=%d: description is not valid UTF-8 (cut inside a rune)", field, pad)
}
}
}
}
23 changes: 12 additions & 11 deletions internal/registries/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@ import (
"strings"

"github.com/smart-mcp-proxy/mcpproxy-go/internal/experiments"
"github.com/smart-mcp-proxy/mcpproxy-go/internal/stringutil"
)

// pulseDescriptionMaxBytes caps the description carried on a Pulse registry
// search result.
const pulseDescriptionMaxBytes = 300

// Sentinel errors for single-server lookup. Surfaces map these to stable error
// codes (registry_not_found / server_not_found) for cross-surface consistency
// (CN-004).
Expand Down Expand Up @@ -702,19 +707,15 @@ func parsePulseWithoutGuesser(rawData interface{}) []ServerEntry {
Name: name,
}

// Try to get description from multiple fields
// Try to get description from multiple fields. The display cap goes
// through stringutil.UTF8Prefix: a registry description is
// upstream-authored free text, and a raw byte cut at 300 can land
// inside a multi-byte rune, leaving the entry invalid UTF-8 (U+FFFD
// once the REST/Web UI encoder sees it).
if shortDesc, ok := itemMap["short_description"].(string); ok && shortDesc != "" {
if len(shortDesc) > 300 {
server.Description = shortDesc[:300]
} else {
server.Description = shortDesc
}
server.Description = stringutil.UTF8Prefix(shortDesc, pulseDescriptionMaxBytes)
} else if aiDesc, ok := itemMap["EXPERIMENTAL_ai_generated_description"].(string); ok && aiDesc != "" {
if len(aiDesc) > 300 {
server.Description = aiDesc[:300]
} else {
server.Description = aiDesc
}
server.Description = stringutil.UTF8Prefix(aiDesc, pulseDescriptionMaxBytes)
} else {
server.Description = noDescAvailable
}
Expand Down
8 changes: 7 additions & 1 deletion internal/storage/activity.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,20 @@ func parseActivityKey(key []byte) string {

// truncateResponse truncates a response string if it exceeds maxSize.
// Returns the (potentially truncated) string and whether truncation occurred.
//
// The cut goes through truncateRuneSafe, the same in-package helper the
// per-server tool-call records use: an upstream response is arbitrary UTF-8,
// and a raw byte cut at maxSize can land inside a multi-byte rune. The record
// is then persisted and served over /api/v1/activities with invalid UTF-8,
// which JSON encoding rewrites to U+FFFD.
func truncateResponse(response string, maxSize int) (string, bool) {
if maxSize <= 0 {
maxSize = DefaultMaxResponseSize
}
if len(response) <= maxSize {
return response, false
}
return response[:maxSize] + "...[truncated]", true
return truncateRuneSafe(response, maxSize) + truncationMarker, true
}

// SaveActivity stores an activity record in BBolt.
Expand Down
45 changes: 45 additions & 0 deletions internal/storage/activity_utf8_truncation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package storage

import (
"strings"
"testing"
"unicode/utf8"
)

// Round 3 of the UTF-8 truncation sweep (#1647, #1664, #1665): the activity
// log's own per-record cut was still a raw byte slice, so a response whose
// cut offset lands inside a multi-byte rune was persisted — and served over
// /api/v1/activities and the CSV export — as invalid UTF-8, which JSON
// encoding rewrites to U+FFFD.
//
// Every offset inside a 3-byte rune is exercised, because only two of the
// three are mid-rune: a single fixed cap would pass by luck.
func TestTruncateResponseKeepsRuneBoundary(t *testing.T) {
body := strings.Repeat("語", 200) // 3 bytes per rune

for _, maxSize := range []int{100, 101, 102, 103, 104, 105} {
got, truncated := truncateResponse(body, maxSize)
if !truncated {
t.Fatalf("maxSize=%d: expected truncation of a %d-byte body", maxSize, len(body))
}
kept := strings.TrimSuffix(got, "...[truncated]")
if !utf8.ValidString(kept) {
t.Errorf("maxSize=%d: stored response ends in a partial rune: % x",
maxSize, []byte(kept)[max(0, len(kept)-4):])
}
if len(kept) > maxSize {
t.Errorf("maxSize=%d: kept %d bytes, which is over the cap", maxSize, len(kept))
}
}
}

// ASCII behaviour must not move: the cap is still applied exactly.
func TestTruncateResponseASCIIUnchanged(t *testing.T) {
got, truncated := truncateResponse(strings.Repeat("x", 500), 100)
if !truncated {
t.Fatal("expected truncation")
}
if want := strings.Repeat("x", 100) + "...[truncated]"; got != want {
t.Errorf("ASCII cut moved:\n got %q\nwant %q", got, want)
}
}
Loading