fix(fal): classify 403 balance-exhausted as insufficient_balance, not auth_failure

When a fal (fal.ai) account runs out of credit, fal returns HTTP 403 with a
body like `{"detail":"User is locked. Reason: TOP_UP."}` or "...Reason:
Exhausted balance. Top up your balance at fal.ai/dashboard/billing.".

The fal adapter previously classified ALL 401/403 as
Scope=ScopeKey / Reason=auth_failure, which maps to a fixed 1-hour
provider-level cooldown (the "bad key, needs human intervention" policy).
That is wrong for an out-of-credit state: the key is valid and recovery is
self-service — after a top-up the account works again immediately, yet maxx
kept the provider cooled for up to an hour.

Fix: detect the billing/lock case in the fal doJSON error classifier — a 403
(or 402) whose `detail` matches billing signals (TOP_UP / "Exhausted balance"
/ "User is locked" / "top up" / insufficient / balance, case-insensitive) —
and classify it with a new dedicated CooldownReasonInsufficientBalance. It
stays ScopeKey and non-retryable (retrying now won't help), but maps to a
short fixed 2-minute cooldown so the provider recovers promptly post top-up,
and the reason string in logs/DB is now "insufficient_balance" instead of the
misleading "auth_failure". Genuine auth failures (401 always; 403 without
billing signals) remain auth_failure / 1h as before.

Adds the CooldownReason enum value in both canonical places (domain and the
cooldown policy package) plus its 2m FixedDurationPolicy, and focused tests
in the fal package (403 TOP_UP/Exhausted vs 401 / plain 403) and the cooldown
package (new reason maps short, shorter than auth_failure).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: Bowl42

internal/adapter/provider/fal/adapter.go

@@ -145,7 +145,16 @@ func (a *Adapter) doJSON(ctx context.Context, method, url string, body []byte) (
 			fmt.Sprintf("fal returned status %d", resp.StatusCode),
 		)
 		proxyErr.HTTPStatusCode = resp.StatusCode
-		if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		if isFalBalanceLocked(resp.StatusCode, respBody) {
+			// fal returns 403 (occasionally 402) with a "User is locked. Reason:
+			// TOP_UP." / "Exhausted balance." detail when the account runs out of
+			// credit. The key is valid and recovery is self-service (top up), so
+			// this must NOT get the heavy 1h auth-failure cooldown — a short
+			// key-scoped cooldown lets the provider recover promptly post top-up.
+			proxyErr.Scope = domain.ScopeKey
+			proxyErr.Reason = domain.CooldownReasonInsufficientBalance
+			proxyErr.Retryable = false
+		} else if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
 			proxyErr.Scope = domain.ScopeKey
 			proxyErr.Reason = domain.CooldownReasonAuthFailure
 			proxyErr.Retryable = false
@@ -161,6 +170,36 @@ func (a *Adapter) doJSON(ctx context.Context, method, url string, body []byte) (
 	return respBody, resp.StatusCode, nil
 }
 
+// falBalanceLockSignals are case-insensitive substrings fal uses in its 4xx
+// `detail` string when an account is out of credit / locked pending a top-up
+// (e.g. `{"detail":"User is locked. Reason: TOP_UP."}` or "...Reason: Exhausted
+// balance. Top up your balance at fal.ai/dashboard/billing."). These are a
+// billing state, NOT a bad/expired key.
+var falBalanceLockSignals = []string{
+	"top_up",
+	"top up",
+	"exhausted balance",
+	"user is locked",
+	"insufficient",
+	"balance",
+}
+
+// isFalBalanceLocked reports whether a fal 4xx response is a billing/lock state
+// (out of credit) rather than a genuine auth failure. Only 402/403 bodies are
+// considered — a 401 is always treated as a real credential problem.
+func isFalBalanceLocked(code int, body []byte) bool {
+	if code != http.StatusForbidden && code != http.StatusPaymentRequired {
+		return false
+	}
+	lower := strings.ToLower(string(body))
+	for _, sig := range falBalanceLockSignals {
+		if strings.Contains(lower, sig) {
+			return true
+		}
+	}
+	return false
+}
+
 func isRetryableStatus(code int) bool {
 	switch code {
 	case 429, 500, 502, 503, 504:

internal/adapter/provider/fal/adapter_test.go

@@ -640,3 +640,108 @@ func TestTaskIDIsURLSafe(t *testing.T) {
 		t.Fatalf("decode round-trip failed: %v", err)
 	}
 }
+
+// ---- Error classification: 403 balance/lock vs genuine auth failure ----
+
+// TestDoJSONBalanceLockedNotAuthFailure guards the real incident: fal returns
+// HTTP 403 with a "User is locked. Reason: TOP_UP." / "Exhausted balance." detail
+// when an account runs out of credit. That must classify as the lighter
+// insufficient_balance reason (short, self-recovering cooldown), NOT the 1h
+// auth_failure cooldown, while genuine credential problems still classify as
+// auth_failure.
+func TestDoJSONBalanceLockedNotAuthFailure(t *testing.T) {
+	cases := []struct {
+		name       string
+		status     int
+		body       string
+		wantScope  domain.ErrorScope
+		wantReason domain.CooldownReason
+	}{
+		{
+			name:       "403 TOP_UP lock",
+			status:     http.StatusForbidden,
+			body:       `{"detail":"User is locked. Reason: TOP_UP."}`,
+			wantScope:  domain.ScopeKey,
+			wantReason: domain.CooldownReasonInsufficientBalance,
+		},
+		{
+			name:       "403 exhausted balance",
+			status:     http.StatusForbidden,
+			body:       `{"detail":"User is locked. Reason: Exhausted balance. Top up your balance at fal.ai/dashboard/billing."}`,
+			wantScope:  domain.ScopeKey,
+			wantReason: domain.CooldownReasonInsufficientBalance,
+		},
+		{
+			name:       "402 insufficient balance",
+			status:     http.StatusPaymentRequired,
+			body:       `{"detail":"insufficient balance"}`,
+			wantScope:  domain.ScopeKey,
+			wantReason: domain.CooldownReasonInsufficientBalance,
+		},
+		{
+			name:       "403 genuine auth failure (no billing signal)",
+			status:     http.StatusForbidden,
+			body:       `{"detail":"Forbidden: invalid key"}`,
+			wantScope:  domain.ScopeKey,
+			wantReason: domain.CooldownReasonAuthFailure,
+		},
+		{
+			name:       "401 always auth failure",
+			status:     http.StatusUnauthorized,
+			body:       `{"detail":"Unauthorized. Reason: TOP_UP."}`, // even with a billing-ish word, 401 stays auth
+			wantScope:  domain.ScopeKey,
+			wantReason: domain.CooldownReasonAuthFailure,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(tc.status)
+				_, _ = w.Write([]byte(tc.body))
+			}))
+			defer server.Close()
+
+			a := newFalAdapter(t)
+			_, status, err := a.doJSON(t.Context(), http.MethodPost, server.URL+"/fal-ai/flux/dev", []byte(`{}`))
+			if status != tc.status {
+				t.Fatalf("status = %d, want %d", status, tc.status)
+			}
+			pe, ok := err.(*domain.ProxyError)
+			if !ok {
+				t.Fatalf("error = %T (%v), want *domain.ProxyError", err, err)
+			}
+			if pe.Scope != tc.wantScope {
+				t.Fatalf("scope = %q, want %q", pe.Scope, tc.wantScope)
+			}
+			if pe.Reason != tc.wantReason {
+				t.Fatalf("reason = %q, want %q", pe.Reason, tc.wantReason)
+			}
+			if pe.Retryable {
+				t.Fatalf("balance/auth errors must be non-retryable, got Retryable=true")
+			}
+		})
+	}
+}
+
+func TestIsFalBalanceLocked(t *testing.T) {
+	cases := []struct {
+		code int
+		body string
+		want bool
+	}{
+		{http.StatusForbidden, `{"detail":"User is locked. Reason: TOP_UP."}`, true},
+		{http.StatusForbidden, `{"detail":"Exhausted balance."}`, true},
+		{http.StatusForbidden, `{"detail":"TOP UP your balance"}`, true},
+		{http.StatusPaymentRequired, `{"detail":"insufficient funds"}`, true},
+		{http.StatusForbidden, `{"detail":"invalid api key"}`, false},
+		{http.StatusUnauthorized, `{"detail":"User is locked. Reason: TOP_UP."}`, false}, // 401 never billing
+		{http.StatusTooManyRequests, `{"detail":"balance"}`, false},                      // wrong status
+	}
+	for _, tc := range cases {
+		if got := isFalBalanceLocked(tc.code, []byte(tc.body)); got != tc.want {
+			t.Fatalf("isFalBalanceLocked(%d, %q) = %v, want %v", tc.code, tc.body, got, tc.want)
+		}
+	}
+}

internal/cooldown/policy.go

@@ -62,15 +62,16 @@ func (p *ExponentialBackoffPolicy) CalculateCooldown(failureCount int) time.Dura
 type CooldownReason string
 
 const (
-	ReasonServerError     CooldownReason = "server_error"          // 5xx errors
-	ReasonNetworkError    CooldownReason = "network_error"         // Connection timeout, DNS failure, etc.
-	ReasonQuotaExhausted  CooldownReason = "quota_exhausted"       // API quota exhausted (fallback when no explicit time)
-	ReasonRateLimit       CooldownReason = "rate_limit_exceeded"   // Rate limit (fallback when no explicit time)
-	ReasonConcurrentLimit CooldownReason = "concurrent_limit"      // Concurrent request limit (fallback when no explicit time)
-	ReasonUnknown          CooldownReason = "unknown"               // Unknown error
-	ReasonAuthFailure      CooldownReason = "auth_failure"          // API key invalid, expired, or account suspended
-	ReasonModelUnavailable CooldownReason = "model_unavailable"     // Model not found or access denied
-	ReasonManual           CooldownReason = "manual"                // Manually frozen by admin
+	ReasonServerError         CooldownReason = "server_error"         // 5xx errors
+	ReasonNetworkError        CooldownReason = "network_error"        // Connection timeout, DNS failure, etc.
+	ReasonQuotaExhausted      CooldownReason = "quota_exhausted"      // API quota exhausted (fallback when no explicit time)
+	ReasonRateLimit           CooldownReason = "rate_limit_exceeded"  // Rate limit (fallback when no explicit time)
+	ReasonConcurrentLimit     CooldownReason = "concurrent_limit"     // Concurrent request limit (fallback when no explicit time)
+	ReasonUnknown             CooldownReason = "unknown"              // Unknown error
+	ReasonAuthFailure         CooldownReason = "auth_failure"         // API key invalid, expired, or account suspended
+	ReasonInsufficientBalance CooldownReason = "insufficient_balance" // Account out of credit / locked pending top-up (recovers on its own after a top-up)
+	ReasonModelUnavailable    CooldownReason = "model_unavailable"    // Model not found or access denied
+	ReasonManual              CooldownReason = "manual"               // Manually frozen by admin
 )
 
 // DefaultPolicies returns the default policy configuration
@@ -109,6 +110,12 @@ func DefaultPolicies() map[CooldownReason]CooldownPolicy {
 		ReasonAuthFailure: &FixedDurationPolicy{
 			Duration: 1 * time.Hour,
 		},
+		// Insufficient balance / account locked pending top-up: fixed 2 minutes.
+		// The key is valid — the account just ran out of credit — so recovery is
+		// self-service (top up) and should be picked up quickly, unlike a bad key.
+		ReasonInsufficientBalance: &FixedDurationPolicy{
+			Duration: 2 * time.Minute,
+		},
 		// Model unavailable: fixed 5 minutes (model might come back)
 		ReasonModelUnavailable: &FixedDurationPolicy{
 			Duration: 5 * time.Minute,

internal/cooldown/policy_test.go

@@ -0,0 +1,28 @@
+package cooldown
+
+import (
+	"testing"
+	"time"
+)
+
+// TestInsufficientBalancePolicyIsShort guards that an out-of-credit / account-
+// locked state (fal "TOP_UP" / "Exhausted balance" 403) recovers quickly: it
+// must map to a short fixed cooldown, NOT the heavy 1h auth-failure cooldown, so
+// the provider comes back promptly after a top-up.
+func TestInsufficientBalancePolicyIsShort(t *testing.T) {
+	policies := DefaultPolicies()
+
+	bal, ok := policies[ReasonInsufficientBalance]
+	if !ok {
+		t.Fatalf("DefaultPolicies missing ReasonInsufficientBalance")
+	}
+	got := bal.CalculateCooldown(1)
+	if got != 2*time.Minute {
+		t.Fatalf("insufficient_balance cooldown = %v, want 2m", got)
+	}
+
+	auth := policies[ReasonAuthFailure].CalculateCooldown(1)
+	if got >= auth {
+		t.Fatalf("insufficient_balance cooldown (%v) must be shorter than auth_failure (%v)", got, auth)
+	}
+}

internal/domain/cooldown.go

@@ -6,14 +6,15 @@ import "time"
 type CooldownReason string
 
 const (
-	CooldownReasonServerError        CooldownReason = "server_error"
-	CooldownReasonNetworkError       CooldownReason = "network_error"
-	CooldownReasonQuotaExhausted     CooldownReason = "quota_exhausted"
-	CooldownReasonRateLimitExceeded  CooldownReason = "rate_limit_exceeded"
-	CooldownReasonConcurrentLimit    CooldownReason = "concurrent_limit"
-	CooldownReasonAuthFailure        CooldownReason = "auth_failure"
-	CooldownReasonModelUnavailable   CooldownReason = "model_unavailable"
-	CooldownReasonUnknown            CooldownReason = "unknown"
+	CooldownReasonServerError         CooldownReason = "server_error"
+	CooldownReasonNetworkError        CooldownReason = "network_error"
+	CooldownReasonQuotaExhausted      CooldownReason = "quota_exhausted"
+	CooldownReasonRateLimitExceeded   CooldownReason = "rate_limit_exceeded"
+	CooldownReasonConcurrentLimit     CooldownReason = "concurrent_limit"
+	CooldownReasonAuthFailure         CooldownReason = "auth_failure"
+	CooldownReasonInsufficientBalance CooldownReason = "insufficient_balance"
+	CooldownReasonModelUnavailable    CooldownReason = "model_unavailable"
+	CooldownReasonUnknown             CooldownReason = "unknown"
 )
 
 // Cooldown represents a provider cooldown record