Merge pull request #24 from gitcommit90/fix/transport-failure-account-lock

Fix transport failure account locks

Author: gitcommit90

CHANGELOG.md

@@ -9,6 +9,12 @@ Release tags use the form `vX.Y.Z` and match `package.json`. GitHub Releases car
 
 ## [Unreleased]
 
+## [0.5.11] - 2026-08-12
+
+### Fixed
+
+- Connection-level failures now retry twice at ten-second intervals, preserve their underlying network cause in logs, advance named-route fallback after exhaustion, and never create misleading OAuth account locks.
+
 ## [0.5.10] - 2026-07-29
 
 ### Fixed
@@ -108,7 +114,8 @@ Release tags use the form `vX.Y.Z` and match `package.json`. GitHub Releases car
 
 See [GitHub Releases](https://github.com/gitcommit90/rerouted/releases) for artifact digests and notes prior to the Keep a Changelog narrative. Notable themes in late 0.4.x included signed/notarized distribution, in-app updates, named routes, OAuth account pools, OpenAI chat completions and Responses routing, and launch hardening.
 
-[Unreleased]: https://github.com/gitcommit90/rerouted/compare/v0.5.10...HEAD
+[Unreleased]: https://github.com/gitcommit90/rerouted/compare/v0.5.11...HEAD
+[0.5.11]: https://github.com/gitcommit90/rerouted/compare/v0.5.10...v0.5.11
 [0.5.10]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.10
 [0.5.9]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.9
 [0.5.8]: https://github.com/gitcommit90/rerouted/releases/tag/v0.5.8

package-lock.json

@@ -1,7 +1,7 @@
 {
   "name": "@gitcommit90/rerouted",
   "productName": "ReRouted",
-  "version": "0.5.10",
+  "version": "0.5.11",
   "description": "A local AI router for connected accounts, models, named routes, and automatic fallback.",
   "author": "gitcommit90",
   "license": "MIT",

package.json

@@ -24,6 +24,8 @@ const COOLDOWN_MS = {
   auth: 2 * 60_000,
   transient: 30_000,
 };
+const TRANSPORT_RETRY_DELAY_MS = 10_000;
+const TRANSPORT_RETRY_ATTEMPTS = 2;
 const PREOUTPUT_INSPECTION_BYTES = 64 * 1024;
 const CLAUDE_CODE_CANONICAL_ROUTES = new Map([
   ["claude-fable-5", "fable"],
@@ -746,6 +748,66 @@ function isAbortError(err) {
   return /aborted|timeout|TimeoutError/i.test(msg);
 }
 
+function transportErrorMessage(error) {
+  const parts = [];
+  const seen = new Set();
+  let current = error;
+  for (let depth = 0; current && depth < 4; depth += 1) {
+    if (seen.has(current)) break;
+    seen.add(current);
+    const code = typeof current.code === "string" ? current.code.trim() : "";
+    const message = String(current.message || current).trim();
+    const detail = code && !message.includes(code) ? `${code}: ${message}` : message;
+    if (detail && !parts.includes(detail)) parts.push(detail);
+    current = current.cause;
+  }
+  return parts.join("; ") || "Upstream connection failed";
+}
+
+function isTransportError(error) {
+  const codes = new Set([
+    "EAI_AGAIN",
+    "ECONNABORTED",
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "EHOSTUNREACH",
+    "ENETDOWN",
+    "ENETUNREACH",
+    "ENOTFOUND",
+    "EPIPE",
+    "ETIMEDOUT",
+    "UND_ERR_CONNECT_TIMEOUT",
+    "UND_ERR_HEADERS_TIMEOUT",
+    "UND_ERR_SOCKET",
+  ]);
+  const seen = new Set();
+  let current = error;
+  for (let depth = 0; current && depth < 4; depth += 1) {
+    if (seen.has(current)) break;
+    seen.add(current);
+    if (codes.has(current.code)) return true;
+    if (/\bfetch failed\b/i.test(String(current.message || ""))) return true;
+    current = current.cause;
+  }
+  return false;
+}
+
+function waitForTransportRetry(ms, signal) {
+  if (signal?.aborted) return Promise.resolve(false);
+  if (!ms || ms <= 0) return Promise.resolve(true);
+  return new Promise((resolve) => {
+    let timer;
+    const finish = (ready) => {
+      if (timer) clearTimeout(timer);
+      signal?.removeEventListener("abort", onAbort);
+      resolve(ready);
+    };
+    const onAbort = () => finish(false);
+    timer = setTimeout(() => finish(true), ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
 function memberSignal(outer, timeoutMs) {
   const ms = timeoutMs ?? REQUEST_TIMEOUT_MS;
   const ctrl = new AbortController();
@@ -790,7 +852,16 @@ function memberSignal(outer, timeoutMs) {
 /**
  * @param {{ store, fetchImpl?, requestLog?, timeoutMs?, usage?, logger? }} opts
  */
-function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, logger = appLogger } = {}) {
+function createRouter({
+  store,
+  fetchImpl = fetch,
+  requestLog,
+  timeoutMs,
+  usage,
+  logger = appLogger,
+  transportRetryDelayMs = TRANSPORT_RETRY_DELAY_MS,
+  transportRetryAttempts = TRANSPORT_RETRY_ATTEMPTS,
+} = {}) {
   const rrState = new Map();
   const log = requestLog || createRequestLog();
   let totalRequests = 0;
@@ -1158,17 +1229,29 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage,
         return {
           ok: false,
           status: 408,
-          error: e.message || "Upstream request timeout",
-          cooldownEligible: true,
-          failureKind: "transient",
-          defaultCooldownMs: COOLDOWN_MS.transient,
+          error: transportErrorMessage(e) || "Upstream request timeout",
+          cooldownEligible: false,
+          failureKind: "transport",
+          defaultCooldownMs: 0,
+          transportError: false,
         };
       }
-      const error = e.message || String(e);
+      const error = transportErrorMessage(e);
       const classification = classifyFailure(502, error);
+      if (classification.kind !== "request" && isTransportError(e)) {
+        return {
+          ok: false,
+          status: 502,
+          error,
+          cooldownEligible: false,
+          failureKind: "transport",
+          defaultCooldownMs: 0,
+          transportError: true,
+        };
+      }
       return {
         ok: false,
-        status: classification.kind === "request" ? 400 : 502,
+        status: 400,
         error,
         cooldownEligible: classification.eligible,
         failureKind: classification.kind,
@@ -1290,19 +1373,55 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage,
           }
         }
         let result;
-        try {
-          result = await tryMember(attemptMember, body, stream, signal);
-        } catch (e) {
-          const status = isAbortError(e) ? 408 : 500;
-          const classification = classifyFailure(status, e.message);
-          result = {
-            ok: false,
-            status,
-            error: e.message || String(e),
-            cooldownEligible: classification.eligible,
-            failureKind: classification.kind,
-            defaultCooldownMs: classification.defaultCooldownMs,
-          };
+        let transportRetry = 0;
+        while (true) {
+          try {
+            result = await tryMember(attemptMember, body, stream, signal);
+          } catch (e) {
+            const status = isAbortError(e) ? 408 : 500;
+            const error = transportErrorMessage(e);
+            const classification = classifyFailure(status, error);
+            const transportError = isTransportError(e);
+            result = {
+              ok: false,
+              status,
+              error,
+              cooldownEligible: transportError ? false : classification.eligible,
+              failureKind: transportError ? "transport" : classification.kind,
+              defaultCooldownMs: transportError ? 0 : classification.defaultCooldownMs,
+              transportError,
+            };
+          }
+          if (!result.transportError || result.canceled || transportRetry >= transportRetryAttempts) {
+            break;
+          }
+          transportRetry += 1;
+          logger.warn("router transport retry scheduled", {
+            event: "transport_retry_scheduled",
+            routeModel: modelId,
+            upstreamModel: member.upstreamModel,
+            providerId: provider.id,
+            providerType: canonicalProviderType(provider.type),
+            providerName: provider.name,
+            accountAlias: provider.accountAlias || null,
+            status: result.status,
+            error: result.error,
+            retryAttempt: transportRetry,
+            maxRetries: transportRetryAttempts,
+            retryDelayMs: transportRetryDelayMs,
+          });
+          if (!(await waitForTransportRetry(transportRetryDelayMs, signal))) {
+            result = {
+              ok: false,
+              status: 499,
+              error: "Client disconnected",
+              cooldownEligible: false,
+              canceled: true,
+              failureKind: "canceled",
+              defaultCooldownMs: 0,
+            };
+            break;
+          }
         }
 
         if (result.ok) {
@@ -1359,15 +1478,22 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage,
           };
         }
 
-        const savedLock = persistModelLock(provider, member.upstreamModel, result);
+        const transportCause = result.transportError ? result.error : null;
+        const publicError = result.transportError
+          ? `Upstream connection failed after ${transportRetryAttempts} retries`
+          : result.error;
+        const savedLock = persistModelLock(provider, member.upstreamModel, {
+          ...result,
+          error: publicError,
+        });
         const failed = {
           providerId: provider.id,
           providerType: canonicalProviderType(provider.type),
           providerName: provider.name,
           accountAlias: provider.accountAlias || null,
           model: member.upstreamModel,
           status: result.status,
-          error: result.error,
+          error: publicError,
           failureKind: result.failureKind,
           retryable: true,
           lockedUntil: savedLock?.until || null,
@@ -1378,6 +1504,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage,
           event: oauthAttempt ? "account_failure" : "route_member_failure",
           routeModel: modelId,
           ...failed,
+          ...(transportCause ? { transportCause } : {}),
         });
         fallbackFrom = failed;
       }

src/lib/router.js

@@ -320,6 +320,96 @@ describe("Claude Code canonical named routes", () => {
 });
 
 describe("same-provider OAuth account fallback", () => {
+  it("retries transport failures without locking the account", async () => {
+    const store = createStore(tmpConfig());
+    store.seed({ providers: [chatgptAccount("prov_a", "token-a", 100)] });
+    const logger = captureLogger();
+    let calls = 0;
+    const router = createRouter({
+      store,
+      logger,
+      transportRetryDelayMs: 0,
+      fetchImpl: async () => {
+        calls += 1;
+        if (calls < 3) {
+          const cause = new Error("getaddrinfo EAI_AGAIN chatgpt.com");
+          cause.code = "EAI_AGAIN";
+          throw new TypeError("fetch failed", { cause });
+        }
+        return responsesSuccessResponse("recovered");
+      },
+    });
+
+    const result = await router.chatCompletions({
+      body: {
+        model: "chatgpt/gpt-5.4",
+        messages: [{ role: "user", content: "hello" }],
+        stream: false,
+      },
+    });
+
+    assert.equal(result.ok, true, JSON.stringify(result.error));
+    assert.equal(calls, 3);
+    assert.deepEqual(store.load().providers[0].modelLocks, {});
+    const retries = logger.entries.filter((entry) => entry.meta?.event === "transport_retry_scheduled");
+    assert.equal(retries.length, 2);
+    assert.match(retries[0].meta.error, /EAI_AGAIN/);
+  });
+
+  it("advances a route after transport retries are exhausted without locking the account", async () => {
+    const store = createStore(tmpConfig());
+    store.seed({
+      providers: [
+        chatgptAccount("prov_chatgpt", "chatgpt-token", 100),
+        claudeAccount("prov_claude", "claude-token", 200),
+      ],
+      combos: [
+        {
+          id: "transport-fallback",
+          name: "transport-fallback",
+          strategy: "fallback",
+          members: [
+            { providerId: "prov_chatgpt", model: "gpt-5.4" },
+            { providerId: "prov_claude", model: "claude-fable-5" },
+          ],
+        },
+      ],
+    });
+    const calls = [];
+    const logger = captureLogger();
+    const router = createRouter({
+      store,
+      logger,
+      transportRetryDelayMs: 0,
+      fetchImpl: async (_url, options) => {
+        const token = authToken(options);
+        calls.push(token);
+        if (token === "chatgpt-token") {
+          const cause = new Error("socket disconnected before TLS handshake");
+          cause.code = "ECONNRESET";
+          throw new TypeError("fetch failed", { cause });
+        }
+        return claudeSuccessResponse("fallback worked");
+      },
+    });
+
+    const result = await router.chatCompletions({
+      body: {
+        model: "transport-fallback",
+        messages: [{ role: "user", content: "hello" }],
+        stream: false,
+      },
+    });
+
+    assert.equal(result.ok, true, JSON.stringify(result.error));
+    assert.equal(result.openAiJson.choices[0].message.content, "fallback worked");
+    assert.deepEqual(calls, ["chatgpt-token", "chatgpt-token", "chatgpt-token", "claude-token"]);
+    assert.deepEqual(store.load().providers.find((provider) => provider.id === "prov_chatgpt").modelLocks, {});
+    const failure = logger.entries.find((entry) => entry.meta?.event === "account_failure");
+    assert.equal(failure.meta.error, "Upstream connection failed after 2 retries");
+    assert.match(failure.meta.transportCause, /ECONNRESET/);
+  });
+
   it("does not turn context failures into cooldowns or a terminal 429", async () => {
     const store = createStore(tmpConfig());
     const quotaLock = {