feat: auto failover APIs with LK Cloud (#686)

* feat: auto failover APIs with LK Cloud

retries in alternative datacenters on 5xx and transport failures

* Add auto failover APIs for LK Cloud

Add auto failover APIs for LK Cloud in livekit-server-sdk.

* fixed formatting

* simplified API

* fmt

* remove failover options

* sip perms check on failover test

* sane default timeouts: 10s for API, 30s for SIP dials

* include dial buffer for SIP

* address comments

* handle whatsapp dial timeout as well

* explicitly set default ringing time

* correctly handle whatsapp dial/accept flow

Author: davidzhao

.changeset/wet-dryers-matter.md

@@ -0,0 +1,5 @@
+---
+"livekit-server-sdk": patch
+---
+
+feat: auto failover APIs with LK Cloud

.github/workflows/test-api.yml

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test API
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  failover:
+    runs-on: ubuntu-latest
+    services:
+      mock-server:
+        image: livekit/test-server:latest
+        ports:
+          - 9999:9999
+          - 10000:10000
+          - 10001:10001
+          - 10002:10002
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+        with:
+          node-version: 24
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Wait for mock server
+        run: |
+          for i in $(seq 1 30); do
+            curl -sf http://127.0.0.1:9999/settings/regions >/dev/null && exit 0
+            sleep 1
+          done
+          echo "mock server did not become ready" && exit 1
+
+      - name: Run API tests
+        run: pnpm --filter="livekit-server-sdk" exec vitest --environment node run test/api

packages/livekit-server-sdk/package.json

@@ -45,7 +45,6 @@
   "dependencies": {
     "@bufbuild/protobuf": "^1.10.1",
     "@livekit/protocol": "^1.48.0",
-    "camelcase-keys": "^9.0.0",
     "jose": "^5.1.2"
   },
   "devDependencies": {

packages/livekit-server-sdk/src/AgentDispatchClient.ts

@@ -40,10 +40,10 @@ export class AgentDispatchClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**

packages/livekit-server-sdk/src/ClientOptions.ts

@@ -10,4 +10,9 @@ export type ClientOptions = {
    * Optional timeout, in seconds, for all server requests
    */
   requestTimeout?: number;
+  /**
+   * Whether to fail over to alternative regions on retryable errors (LiveKit
+   * Cloud hosts only). Defaults to true; set to false to disable.
+   */
+  failover?: boolean;
 };

packages/livekit-server-sdk/src/ConnectorClient.ts

@@ -23,6 +23,7 @@ import {
 import type { ClientOptions } from './ClientOptions.js';
 import { ServiceBase } from './ServiceBase.js';
 import { type Rpc, TwirpRpc, livekitPackage } from './TwirpRPC.js';
+import { DEFAULT_RINGING_TIMEOUT_SECONDS } from './dialTimeout.js';
 
 const svc = 'Connector';
 
@@ -87,6 +88,12 @@ export interface AcceptWhatsAppCallOptions {
   ringingTimeout?: number;
   /** Optional - Wait for the call to be answered before returning */
   waitUntilAnswered?: boolean;
+  /**
+   * Optional - Request timeout in seconds. When `waitUntilAnswered` is set,
+   * defaults to a longer value (dialing takes time) and is raised, if needed,
+   * to stay above `ringingTimeout`; otherwise the client default applies.
+   */
+  timeout?: number;
 }
 
 // Twilio types
@@ -123,10 +130,10 @@ export class ConnectorClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**
@@ -206,11 +213,21 @@ export class ConnectorClient extends ServiceBase {
       waitUntilAnswered: options.waitUntilAnswered,
     }).toJson();
 
+    // Accept can block until the call is answered, so default the request timeout
+    // to the standard ring window. The caller overrides it via `timeout` and
+    // should set it above the ringing_timeout passed to dialWhatsAppCall; the
+    // two calls are separate, so the SDK can't derive it. Non-waiting returns
+    // promptly and uses the client default.
+    const timeout = options.waitUntilAnswered
+      ? (options.timeout ?? DEFAULT_RINGING_TIMEOUT_SECONDS)
+      : options.timeout;
+
     const data = await this.rpc.request(
       svc,
       'AcceptWhatsAppCall',
       req,
       await this.authHeader({ roomCreate: true }),
+      timeout,
     );
     return AcceptWhatsAppCallResponse.fromJson(data, { ignoreUnknownFields: true });
   }

packages/livekit-server-sdk/src/EgressClient.ts

@@ -142,10 +142,10 @@ export class EgressClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**

packages/livekit-server-sdk/src/IngressClient.ts

@@ -129,10 +129,10 @@ export class IngressClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**

packages/livekit-server-sdk/src/RoomServiceClient.ts

@@ -132,10 +132,10 @@ export class RoomServiceClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**

packages/livekit-server-sdk/src/SipClient.ts

@@ -46,6 +46,7 @@ import type { ClientOptions } from './ClientOptions.js';
 import { ServiceBase } from './ServiceBase.js';
 import type { Rpc } from './TwirpRPC.js';
 import { TwirpRpc, livekitPackage } from './TwirpRPC.js';
+import { DEFAULT_RINGING_TIMEOUT_SECONDS, dialRequestTimeout } from './dialTimeout.js';
 
 const svc = 'SIP';
 
@@ -170,7 +171,7 @@ export interface CreateSipParticipantOptions {
   krispEnabled?: boolean;
   /** If `true`, this will wait until the call is answered before returning. */
   waitUntilAnswered?: boolean;
-  /** Optional request timeout in seconds. default 60 seconds if waitUntilAnswered is true, otherwise 10 seconds */
+  /** Optional request timeout in seconds. Defaults to 30s when waitUntilAnswered is true (dialing takes time), otherwise the client default. */
   timeout?: number;
   media?: SIPMediaConfig;
 }
@@ -233,6 +234,8 @@ export interface TransferSipParticipantOptions {
   headers?: { [key: string]: string };
   /** Maximum time for the transfer destination to answer the call, in seconds. */
   ringingTimeout?: number;
+  /** Optional request timeout in seconds. Defaults to 30s (dialing takes time). */
+  timeout?: number;
 }
 
 /**
@@ -249,10 +252,10 @@ export class SipClient extends ServiceBase {
    */
   constructor(host: string, apiKey?: string, secret?: string, options?: ClientOptions) {
     super(apiKey, secret);
-    const rpcOptions = options?.requestTimeout
-      ? { requestTimeout: options.requestTimeout }
-      : undefined;
-    this.rpc = new TwirpRpc(host, livekitPackage, rpcOptions);
+    this.rpc = new TwirpRpc(host, livekitPackage, {
+      requestTimeout: options?.requestTimeout,
+      failover: options?.failover,
+    });
   }
 
   /**
@@ -764,8 +767,13 @@ export class SipClient extends ServiceBase {
       opts = {};
     }
 
-    if (opts.timeout === undefined) {
-      opts.timeout = opts.waitUntilAnswered ? 60 : 10;
+    // Dialing a phone and waiting for an answer takes longer than a normal call,
+    // and the request must outlast ringing so the call can be answered. Pin the
+    // ring window explicitly so our request timeout doesn't depend on the server's
+    // default (which could change out from under us).
+    if (opts.waitUntilAnswered) {
+      opts.ringingTimeout ??= DEFAULT_RINGING_TIMEOUT_SECONDS;
+      opts.timeout = dialRequestTimeout(opts.timeout, opts.ringingTimeout);
     }
 
     const req = new CreateSIPParticipantRequest({
@@ -823,6 +831,12 @@ export class SipClient extends ServiceBase {
       opts = {};
     }
 
+    // Transferring a call dials a phone, which takes longer than a normal call,
+    // so keep the request alive past ringing. Pin the ring window explicitly so
+    // our request timeout doesn't depend on the server's default.
+    opts.ringingTimeout ??= DEFAULT_RINGING_TIMEOUT_SECONDS;
+    opts.timeout = dialRequestTimeout(opts.timeout, opts.ringingTimeout);
+
     const req = new TransferSIPParticipantRequest({
       participantIdentity: participantIdentity,
       roomName: roomName,
@@ -839,6 +853,7 @@ export class SipClient extends ServiceBase {
       'TransferSIPParticipant',
       req,
       await this.authHeader({ roomAdmin: true, room: roomName }, { call: true }),
+      opts.timeout,
     );
   }
 }