From 1433c0e7a26340062be516d31296b2cabfd506ad Mon Sep 17 00:00:00 2001 From: Marketen Date: Wed, 4 Nov 2026 15:49:01 +0100 Subject: [PATCH] feat: guest chat requests always carry an output limit Guest chat accounts (is_guest, from Metering) are landing-page visitors with a small credit and no login. Their requests are ordinary requests with one difference: a chat request always carries an output limit of at most GUEST_MAX_OUTPUT_TOKENS (default 8,192), written into the request so the provider enforces it. Metering holds credit for that limit, so a guest's answer can never cost more than its hold; without a limit the request would go upstream unbounded and could leave a debt nobody pays. 8,192 covers almost every answer, reasoning included: over a week of production traffic on DeepSeek V4.1 Flash, answers averaged about 1,100 output tokens and the longest was about 15,000. A 1,024 limit left hard questions with an empty answer, all of it spent on reasoning. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 ++ apps/gateway/guest_env_test.go | 20 +++++ .../adapters/metering/runtime_client.go | 2 + .../adapters/metering/runtime_client_test.go | 26 ++++++ .../application/services/generate_service.go | 10 +++ .../application/services/guest_output.go | 47 ++++++++++ .../application/services/guest_output_test.go | 88 +++++++++++++++++++ .../internal/application/services/proxy.go | 7 ++ apps/gateway/main.go | 20 +++++ pkg/domain/account.go | 4 + 10 files changed, 232 insertions(+) create mode 100644 apps/gateway/guest_env_test.go create mode 100644 apps/gateway/internal/application/services/guest_output.go create mode 100644 apps/gateway/internal/application/services/guest_output_test.go diff --git a/README.md b/README.md index d6eb701..5461dc6 100644 --- a/README.md +++ b/README.md @@ -138,3 +138,11 @@ PII masking covers conversation text, tool results, assistant reasoning, tool-ca argument string values, and `user`. Tool definitions, names, IDs, schemas, metadata, provider options, and image/audio content are not scanned. Keep personal data out of those fields even when masking is enabled. + +Guest chat accounts (flagged `is_guest` by Metering; a landing-page visitor +with a small credit and no login) are ordinary accounts with one difference: +their chat requests always carry an output limit of at most +`GUEST_MAX_OUTPUT_TOKENS` (default 8192), written into the request so the +provider enforces it. Metering holds credit for that limit, so a guest's answer +can never cost more than its hold. An invalid value stops the gateway at +startup. The Nexus repository's `docs/guest-chat.md` describes the design. diff --git a/apps/gateway/guest_env_test.go b/apps/gateway/guest_env_test.go new file mode 100644 index 0000000..f20dd51 --- /dev/null +++ b/apps/gateway/guest_env_test.go @@ -0,0 +1,20 @@ +package main + +import "testing" + +func TestPositiveIntEnv(t *testing.T) { + t.Setenv("GUEST_MAX_OUTPUT_TOKENS", "") + if n, err := positiveIntEnv("GUEST_MAX_OUTPUT_TOKENS", 8192); err != nil || n != 8192 { + t.Fatalf("unset = %d, %v; want the default", n, err) + } + t.Setenv("GUEST_MAX_OUTPUT_TOKENS", "4096") + if n, err := positiveIntEnv("GUEST_MAX_OUTPUT_TOKENS", 8192); err != nil || n != 4096 { + t.Fatalf("4096 = %d, %v", n, err) + } + for _, bad := range []string{"0", "-1", "8k"} { + t.Setenv("GUEST_MAX_OUTPUT_TOKENS", bad) + if _, err := positiveIntEnv("GUEST_MAX_OUTPUT_TOKENS", 8192); err == nil { + t.Fatalf("%q was accepted", bad) + } + } +} diff --git a/apps/gateway/internal/adapters/metering/runtime_client.go b/apps/gateway/internal/adapters/metering/runtime_client.go index ba39c2d..13df452 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client.go +++ b/apps/gateway/internal/adapters/metering/runtime_client.go @@ -33,6 +33,7 @@ func (c *Client) AuthenticateAPIKey(ctx context.Context, rawKey string) (domain. ID: response.Account.ID, ExternalCustomerID: response.Account.ExternalCustomerID, Status: response.Account.Status, + IsGuest: response.Account.IsGuest, }, APIKey: domain.APIKey{ ID: response.APIKey.ID, @@ -413,6 +414,7 @@ type runtimeAccount struct { ID string `json:"id"` ExternalCustomerID *string `json:"external_customer_id,omitempty"` Status string `json:"status"` + IsGuest bool `json:"is_guest"` } type runtimeAPIKey struct { diff --git a/apps/gateway/internal/adapters/metering/runtime_client_test.go b/apps/gateway/internal/adapters/metering/runtime_client_test.go index 2ad1ca5..5f20ce8 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client_test.go +++ b/apps/gateway/internal/adapters/metering/runtime_client_test.go @@ -52,6 +52,32 @@ func TestRuntimeClientAuthenticateHashesAPIKeyLocally(t *testing.T) { } } +// The guest flag decides whether the guest output limit applies, so it must +// survive the trip from metering; an older metering that omits it means "not +// a guest". +func TestRuntimeClientAuthenticateReadsTheGuestFlag(t *testing.T) { + for _, tc := range []struct { + account string + want bool + }{ + {`{"id":"acc-1","status":"active","is_guest":true}`, true}, + {`{"id":"acc-1","status":"active","is_guest":false}`, false}, + {`{"id":"acc-1","status":"active"}`, false}, + } { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"account":` + tc.account + `,"api_key":{"id":"key-1","account_id":"acc-1","key_prefix":"nxs_1234","active":true,"pii_mode":"off"}}`)) + })) + auth, err := NewClient(server.URL, "token", time.Second).AuthenticateAPIKey(context.Background(), "sk-x") + server.Close() + if err != nil { + t.Fatalf("AuthenticateAPIKey: %v", err) + } + if auth.Account.IsGuest != tc.want { + t.Fatalf("account %s: IsGuest = %v, want %v", tc.account, auth.Account.IsGuest, tc.want) + } + } +} + func TestRuntimeClientPreservesUserAuthenticationErrors(t *testing.T) { tests := []struct { name string diff --git a/apps/gateway/internal/application/services/generate_service.go b/apps/gateway/internal/application/services/generate_service.go index feeac0f..9971a19 100644 --- a/apps/gateway/internal/application/services/generate_service.go +++ b/apps/gateway/internal/application/services/generate_service.go @@ -26,6 +26,8 @@ type GenerateService struct { piiFailOpen bool tinfoilProofs ports.TinfoilTransportProofRepository logger ports.Logger + // guestMaxOutputTokens is the output limit of guest chat requests. + guestMaxOutputTokens int } func NewGenerateService( @@ -46,6 +48,8 @@ func NewGenerateService( pii: pii, piiLang: "en", logger: logger, + + guestMaxOutputTokens: DefaultGuestMaxOutputTokens, } } @@ -58,6 +62,12 @@ func (s *GenerateService) SetPIIOptions(language string, failOpen bool) { s.piiFailOpen = failOpen } +// SetGuestMaxOutputTokens sets the output limit of guest chat requests; see +// capGuestOutput. +func (s *GenerateService) SetGuestMaxOutputTokens(limit int) { + s.guestMaxOutputTokens = limit +} + func (s *GenerateService) SetTinfoilProofRepository(proofs ports.TinfoilTransportProofRepository) { s.tinfoilProofs = proofs } diff --git a/apps/gateway/internal/application/services/guest_output.go b/apps/gateway/internal/application/services/guest_output.go new file mode 100644 index 0000000..8395650 --- /dev/null +++ b/apps/gateway/internal/application/services/guest_output.go @@ -0,0 +1,47 @@ +package services + +import ( + "bytes" + "encoding/json" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// DefaultGuestMaxOutputTokens is the output limit of a guest's chat request +// unless GUEST_MAX_OUTPUT_TOKENS sets another. It covers almost every answer, +// reasoning included: over a week of production traffic on DeepSeek V4.1 +// Flash, answers averaged about 1,100 output tokens and the longest was about +// 15,000 (2026-11-04). +const DefaultGuestMaxOutputTokens = 8192 + +// capGuestOutput writes an output limit into a guest's chat request. A guest +// is an ordinary account with a small credit and no way to pay a debt, so an +// answer must never cost more than its hold. Metering holds credit for the +// request's output limit, and the provider enforces the limit written here; +// without one, the request would go upstream unbounded and could overrun the +// hold (docs/METERING_HOLDS.md in the Nexus repo). A smaller limit the guest +// set is kept. Everything else about a guest's request is untouched. +func capGuestOutput(raw []byte, req domain.GenerateRequest, limit int) ([]byte, domain.GenerateRequest, error) { + if req.MaxOutputTokens != nil && *req.MaxOutputTokens > 0 && *req.MaxOutputTokens < limit { + limit = *req.MaxOutputTokens + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() // Forward numbers exactly as sent. + var body map[string]any + if err := dec.Decode(&body); err != nil || body == nil { + return nil, req, domain.ErrInvalidField("invalid JSON body") + } + // Keep the client's field name; providers that need max_tokens get it from + // the provider adapter. The mapper refuses requests that set both. + if _, ok := body["max_completion_tokens"]; ok { + body["max_completion_tokens"] = limit + } else { + body["max_tokens"] = limit + } + rewritten, err := json.Marshal(body) + if err != nil { + return nil, req, domain.ErrInternal("an internal error occurred").WithMeta("reason", "re-encode guest request", "error", err.Error()) + } + req.MaxOutputTokens = &limit + return rewritten, req, nil +} diff --git a/apps/gateway/internal/application/services/guest_output_test.go b/apps/gateway/internal/application/services/guest_output_test.go new file mode 100644 index 0000000..0281e2e --- /dev/null +++ b/apps/gateway/internal/application/services/guest_output_test.go @@ -0,0 +1,88 @@ +package services + +import ( + "context" + "encoding/json" + "testing" + + "github.com/dappnode/dappnode-nexus-gateway/apps/gateway/internal/adapters/http/mapper" + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +func newGuestTestService(meter *stubUsageMeter, provider *stubProvider, isGuest bool) *GenerateService { + svc := newDirectModelGenerateService(meter, provider) + svc.auth = &stubAuthService{authCtx: domain.AuthContext{ + Account: domain.Account{ID: "acc1", Status: domain.AccountStatusActive, IsGuest: isGuest}, + APIKey: domain.APIKey{ID: "key1", Active: true}, + }} + svc.SetGuestMaxOutputTokens(50) + return svc +} + +func proxyChat(t *testing.T, svc *GenerateService, body string) { + t.Helper() + req, err := mapper.ChatCompletionRequestToDomain([]byte(body)) + if err != nil { + t.Fatalf("map request: %v", err) + } + if _, err := svc.Proxy(context.Background(), domain.EndpointChatCompletions, []byte(body), req, "sk-test"); err != nil { + t.Fatalf("Proxy: %v", err) + } +} + +func sentField(t *testing.T, raw, field string) any { + t.Helper() + var body map[string]any + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("provider body: %v", err) + } + return body[field] +} + +// A guest's request always reaches the provider with an output limit no +// larger than the guest limit, and Metering holds credit for that limit. +func TestProxy_GuestRequestsCarryTheOutputLimit(t *testing.T) { + const user = `"messages":[{"role":"user","content":"hi"}]` + for name, tc := range map[string]struct { + body, field string + want float64 + }{ + "no limit": {`{"model":"model",` + user + `}`, "max_tokens", 50}, + "larger limit": {`{"model":"model",` + user + `,"max_tokens":4000}`, "max_tokens", 50}, + "smaller limit is kept": {`{"model":"model",` + user + `,"max_tokens":20}`, "max_tokens", 20}, + "max_completion_tokens name": {`{"model":"model",` + user + `,"max_completion_tokens":4000}`, "max_completion_tokens", 50}, + } { + t.Run(name, func(t *testing.T) { + meter, provider := &stubUsageMeter{}, &stubProvider{} + proxyChat(t, newGuestTestService(meter, provider, true), tc.body) + if got := sentField(t, provider.lastRaw, tc.field); got != tc.want { + t.Fatalf("provider got %s = %v, want %v", tc.field, got, tc.want) + } + if got := meter.lastReserveReq.MaxOutputTokens; got == nil || float64(*got) != tc.want { + t.Fatalf("held output tokens = %v, want %v", got, tc.want) + } + }) + } +} + +// Apart from the output limit, a guest's request is an ordinary request: +// other fields reach the provider as sent, and signed-in accounts get no +// limit added. +func TestProxy_GuestRequestsAreOtherwiseOrdinary(t *testing.T) { + body := `{"model":"model","messages":[{"role":"user","content":"hi"}],"tools":[{"type":"function","function":{"name":"f","parameters":{"type":"object"}}}]}` + + meter, provider := &stubUsageMeter{}, &stubProvider{} + proxyChat(t, newGuestTestService(meter, provider, true), body) + if sentField(t, provider.lastRaw, "tools") == nil { + t.Fatal("a guest's tools were dropped") + } + + meter, provider = &stubUsageMeter{}, &stubProvider{} + proxyChat(t, newGuestTestService(meter, provider, false), body) + if got := sentField(t, provider.lastRaw, "max_tokens"); got != nil { + t.Fatalf("a signed-in request got max_tokens %v", got) + } + if meter.lastReserveReq.MaxOutputTokens != nil { + t.Fatalf("a signed-in request's hold got an output limit: %d", *meter.lastReserveReq.MaxOutputTokens) + } +} diff --git a/apps/gateway/internal/application/services/proxy.go b/apps/gateway/internal/application/services/proxy.go index b7d5982..33883ae 100644 --- a/apps/gateway/internal/application/services/proxy.go +++ b/apps/gateway/internal/application/services/proxy.go @@ -56,6 +56,13 @@ func (s *GenerateService) Proxy(ctx context.Context, endpoint string, raw []byte } return domain.Annotate(err, "account_id", authCtx.Account.ID, "model", model.PublicModelID, "provider", model.ProviderConfig.ProviderName) } + if authCtx.Account.IsGuest && endpoint == domain.EndpointChatCompletions { + if raw, req, err = capGuestOutput(raw, req, s.guestMaxOutputTokens); err != nil { + s.recordTerminalOutcome(metrics.OutcomeError, endpoint, req, nil, time.Since(start).Milliseconds()) + s.recordError(endpoint, req, nil, metrics.PhaseRequest, err) + return nil, reject(err, nil) + } + } var model domain.PublicModel execReq := req if endpoint == domain.EndpointSystemOne { diff --git a/apps/gateway/main.go b/apps/gateway/main.go index 182abf1..e1bc1a5 100644 --- a/apps/gateway/main.go +++ b/apps/gateway/main.go @@ -3,6 +3,7 @@ package main import ( "context" "crypto/rand" + "fmt" "log" "net/http" "os" @@ -93,6 +94,11 @@ func main() { generateSvc := services.NewGenerateService(authService, modelCatalog, routerClient, providerRegistry, meteringClient, piiFilter, zapLogger) generateSvc.SetPIIOptions(piiLang, piiFailOpen) generateSvc.SetTinfoilProofRepository(tinfoilProofRepo) + guestMaxOutput, err := positiveIntEnv("GUEST_MAX_OUTPUT_TOKENS", services.DefaultGuestMaxOutputTokens) + if err != nil { + log.Fatalf("invalid guest chat configuration: %v", err) + } + generateSvc.SetGuestMaxOutputTokens(guestMaxOutput) chatCompletionsSvc := services.NewChatCompletionsService(generateSvc, zapLogger) healthHandler := handlers.NewHealthHandler() @@ -247,3 +253,17 @@ func buildPIIFilter(log *logger.ZapLogger) (ports.PIIFilter, string, bool) { Logger: log, }), language, failOpen } + +// positiveIntEnv reads a positive whole number from the environment. A set but +// invalid value is an error, so a typo cannot silently loosen a limit. +func positiveIntEnv(name string, fallback int) (int, error) { + v := strings.TrimSpace(os.Getenv(name)) + if v == "" { + return fallback, nil + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%s must be a positive whole number", name) + } + return n, nil +} diff --git a/pkg/domain/account.go b/pkg/domain/account.go index 3a50b99..f0be7ce 100644 --- a/pkg/domain/account.go +++ b/pkg/domain/account.go @@ -30,6 +30,10 @@ type Account struct { // LastUsedAt is the most recent `last_used_at` across the account's API keys. // Populated only by list queries that aggregate key usage; nil elsewhere. LastUsedAt *time.Time + // IsGuest marks a guest chat account: a landing-page visitor without a + // login, funded with a small credit. Its chat requests always carry an + // output limit (see services.capGuestOutput). + IsGuest bool } // IsActive returns true if the account status is "active".