remote.test.ts

import { mkdirSync, mkdtempSync, realpathSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
  createManagedFetch,
  DEFAULT_NETWORK_POLICY,
  PiShipError,
} from "@piship/contracts";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { activateSandbox, describeContainment } from "./activate.js";
import type { SandboxPolicy, SandboxProfile } from "./profile.js";
import {
  connectEnvelope,
  E2bCompatibleBackend,
  type E2bCompatibleOptions,
  EnvelopeReader,
} from "./remote/e2b.js";
import {
  KubernetesAgentSandboxBackend,
  type KubernetesAgentSandboxOptions,
  runtimeCommand,
} from "./remote/kubernetes.js";
import { e2bServer, type StartRequest } from "./testing/e2b-server.js";
import {
  CLAIMS,
  type Cluster,
  kubernetesServer,
} from "./testing/kubernetes-server.js";
import {
  closeMockServers,
  leaks,
  type Recorded,
} from "./testing/mock-server.js";

const fetch = createManagedFetch(DEFAULT_NETWORK_POLICY, "sandbox");

const policy = (overrides: Partial<SandboxPolicy> = {}): SandboxPolicy => ({
  required: true,
  filesystem: { read: { deny: ["~/.ssh"] }, write: { allow: ["workspace"] } },
  network: { mode: "deny" },
  environment: {
    allow: ["PATH", "HOME", "LANG", "DOCS_MODE", "ACME_API_TOKEN"],
  },
  ...overrides,
});

const SECRETS = {
  ACME_API_TOKEN: "sk-live-never-in-remote-backends",
  AWS_SECRET_ACCESS_KEY: "aws-never-in-remote-backends",
  GITHUB_TOKEN: "ghp_never_in_remote_backends_0000000000",
};
const ENV = {
  PATH: "/opt/host/bin",
  HOME: "/home/alice",
  LANG: "C.UTF-8",
  DOCS_MODE: "demo",
  UNLISTED_VARIABLE: "unlisted-host-value",
  ...SECRETS,
};

let root: string;
let workspace: string;
beforeEach(() => {
  root = realpathSync(mkdtempSync(join(tmpdir(), "piship-remote-")));
  workspace = join(root, "ws");
  mkdirSync(join(workspace, "sub"), { recursive: true });
});
afterEach(async () => {
  await closeMockServers();
  rmSync(root, { recursive: true, force: true });
});

// ------------------------------------------------------------------ E2B

function e2b(url: string, options: Partial<E2bCompatibleOptions> = {}) {
  return new E2bCompatibleBackend({
    endpoint: url,
    fetch,
    template: "piship-workspace",
    envdUrl: () => url,
    ...options,
  });
}

const activate = (
  backend: E2bCompatibleBackend | KubernetesAgentSandboxBackend,
  config: SandboxPolicy = policy(
    backend instanceof KubernetesAgentSandboxBackend
      ? { network: { mode: "allow" } }
      : {},
  ),
) =>
  activateSandbox(config, {
    workspace,
    homeDir: join(root, "home"),
    backend,
    env: ENV,
    settleMs: 2000,
  });

const run = async (
  sandbox: Awaited<ReturnType<typeof activateSandbox>>,
  command: string,
  options: { cwd?: string; timeout?: number } = {},
) => {
  let output = "";
  const result = await sandbox.exec(command, options.cwd ?? workspace, {
    onData: (chunk) => {
      output += chunk.toString("utf8");
    },
    ...(options.timeout ? { timeout: options.timeout } : {}),
  });
  return { ...result, output };
};

const starts = (requests: Recorded[]) =>
  requests
    .filter((request) => request.path === "/process.Process/Start")
    .map(
      (request) =>
        new EnvelopeReader().push(request.body)[0]?.message as StartRequest,
    );

describe("remote capability reporting", () => {
  const backends = () => [
    e2b("http://127.0.0.1:1"),
    kubernetes("http://127.0.0.1:1"),
  ];

  it("never declares PiShip's filesystem path planes for a remote backend", () => {
    for (const backend of backends()) {
      const capabilities = backend.capabilities();
      expect(capabilities.isolation).toBe("remote");
      expect(capabilities.planes).toEqual(
        backend instanceof KubernetesAgentSandboxBackend
          ? ["host-filesystem-isolation", "environment-filter"]
          : ["host-filesystem-isolation", "network-deny", "environment-filter"],
      );
      expect(capabilities.planes).not.toContain("filesystem-read-deny");
      expect(capabilities.planes).not.toContain("filesystem-write-allowlist");
      expect(capabilities.localProcesses).toBe(false);
      // Neither backend knows the developer's workspace: a template or a
      // warm-pool pod holds a copy of the code at best.
      expect(capabilities.workspace).toEqual({ mode: "snapshot" });
    }
  });

  it("refuses Kubernetes network deny without authoritative NetworkPolicy evidence", async () => {
    const mock = await kubernetesServer();
    await expect(
      activate(kubernetes(mock.url), policy()),
    ).rejects.toMatchObject({
      code: "SANDBOX_UNAVAILABLE",
    });
    expect(
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === CLAIMS,
      ),
    ).toHaveLength(0);
  });

  it("reports e2b network denial attested: no probe, one sandbox, no connection attempt", async () => {
    const mock = await e2bServer();
    const backend = e2b(mock.url);
    expect(backend.capabilities().networkProbe).toBeUndefined();
    const sandbox = await activate(backend);
    expect(sandbox.report.networkDenial).toEqual({
      evidence: "attested",
      probe: false,
      reason: "the backend declares no network probe to check it with",
    });
    const created = mock.requests.filter(
      (request) => request.method === "POST" && request.path === "/sandboxes",
    );
    expect(created).toHaveLength(1);
    expect(JSON.parse(created[0]?.body.toString() ?? "{}")).toMatchObject({
      allow_internet_access: false,
    });
    for (const request of mock.requests)
      expect(request.body.toString("latin1")).not.toContain("/dev/tcp/");
    await sandbox.dispose();
  });

  it("reports only enforced guarantees in the containment report and doctor line", async () => {
    const mock = await e2bServer();
    const sandbox = await activate(e2b(mock.url));
    expect(sandbox.report.planes).toEqual([
      "network-deny",
      "environment-filter",
      "host-filesystem-isolation",
    ]);
    const line = describeContainment(sandbox.report);
    expect(line).toBe(
      "enforced by e2b-compatible (required, attested by the backend): network-deny, environment-filter, host-filesystem-isolation; network deny (attested by the backend, not verified). Contains shell commands; MCP stdio servers cannot be contained by this backend and do not start, not the agent process or in-process extensions. The sandbox cannot reach this host's files, but it does not enforce sandbox.filesystem path rules; they govern only the local file tools. Workspace: snapshot. Remote commands see a copy, not the files the agent edits; this is not a complete coding-agent workspace.",
    );
    expect(sandbox.report).toMatchObject({
      isolation: "remote",
      workspace: {
        declared: "snapshot",
        effective: "snapshot",
        verification: "not-required",
        gitControlProtection: "not-applicable",
        complete: false,
      },
    });
    // A snapshot is never checked: no sentinel reaches the service.
    expect((await run(sandbox, "true")).exitCode).toBe(0);
    expect(sandbox.workspace()?.verification).toBe("not-required");
    const wire = mock.requests.map((request) => request.body.toString());
    expect(wire.join("\n")).not.toMatch(/piship-ws|\.git\/piship-workspace/);
    expect(line).not.toMatch(/filesystem-read-deny|filesystem-write-allowlist/);
    await sandbox.dispose();
  });
});

describe("e2b-compatible backend against a mock server", () => {
  it("creates, checks, runs, and deletes a sandbox", async () => {
    const mock = await e2bServer({
      command: (start) => ({
        stdout: `cwd=${start.process.cwd}\n`,
        exitCode: start.process.args[2] === "exit 3" ? 3 : 0,
      }),
    });
    const sandbox = await activate(e2b(mock.url));
    expect(sandbox.report).toMatchObject({
      level: "enforced",
      adapter: "e2b-compatible",
      provider: "e2b-compatible",
      verification: "backend-attested",
      localProcesses: false,
    });
    expect(await run(sandbox, "pwd", { cwd: join(workspace, "sub") })).toEqual({
      exitCode: 0,
      output: "cwd=/home/user/sub\n",
    });
    expect((await run(sandbox, "exit 3")).exitCode).toBe(3);
    await sandbox.dispose();
    const created = JSON.parse(
      mock.requests
        .find((request) => request.path === "/sandboxes")
        ?.body.toString() ?? "{}",
    );
    expect(created).toEqual({
      templateID: "piship-workspace",
      timeout: 3600,
      metadata: { "created-by": "piship" },
      allow_internet_access: false,
    });
    const start = starts(mock.requests).at(-1);
    expect(start?.process.cmd).toBe("/bin/bash");
    expect(start?.process.args).toEqual(["-l", "-c", "exit 3"]);
    expect(start?.stdin).toBe(false);
    expect(
      mock.requests.filter(
        (request) => request.path === "/sandboxes/sbx1/timeout",
      ),
    ).toHaveLength(3);
    expect(mock.requests.at(-1)).toMatchObject({
      method: "DELETE",
      path: "/sandboxes/sbx1",
    });
  });

  it("maps the workspace onto a configured workdir and allows the network only when asked", async () => {
    const mock = await e2bServer({
      command: (start) => ({ stdout: `${start.process.cwd}\n` }),
    });
    const sandbox = await activate(
      e2b(mock.url, { workdir: "/workspace/repo" }),
      policy({ network: { mode: "allow" } }),
    );
    expect((await run(sandbox, "pwd")).output).toBe("/workspace/repo\n");
    expect(
      (await run(sandbox, "pwd", { cwd: join(workspace, "sub") })).output,
    ).toBe("/workspace/repo/sub\n");
    await sandbox.dispose();
    const created = JSON.parse(
      mock.requests
        .find((request) => request.path === "/sandboxes")
        ?.body.toString() ?? "{}",
    );
    expect(created.allow_internet_access).toBe(true);
  });

  it("sends only approved, non-host variables and no credentials", async () => {
    const mock = await e2bServer();
    const sandbox = await activate(e2b(mock.url));
    await run(sandbox, "env");
    await sandbox.dispose();
    expect(starts(mock.requests).at(-1)?.process.envs).toEqual({
      LANG: "C.UTF-8",
      DOCS_MODE: "demo",
    });
    const wire = JSON.stringify(
      mock.requests.map((request) => ({
        headers: request.headers,
        body: request.body.toString("latin1"),
      })),
    );
    for (const value of [
      ...Object.values(SECRETS),
      "unlisted-host-value",
      "/home/alice",
      "/opt/host/bin",
    ])
      expect(wire).not.toContain(value);
  });

  it("sends the endpoint credential to the control plane only", async () => {
    const mock = await e2bServer();
    const sandbox = await activate(
      e2b(mock.url, { credential: async () => "runtime-credential-1" }),
    );
    await run(sandbox, "true");
    await sandbox.dispose();
    for (const request of mock.requests) {
      const control =
        request.path.startsWith("/sandboxes") && request.path !== "/health";
      if (control)
        expect(request.headers["x-api-key"]).toBe("runtime-credential-1");
      else
        expect(JSON.stringify(request.headers)).not.toContain(
          "runtime-credential-1",
        );
      if (request.path.startsWith("/process.Process/")) {
        expect(request.headers["x-access-token"]).toBe("envd-token-1");
        expect(request.headers["e2b-sandbox-id"]).toBe("sbx1");
        expect(request.headers["e2b-sandbox-port"]).toBe("49983");
        expect(request.headers["connect-protocol-version"]).toBe("1");
      }
    }
  });

  it("sends a required API key only as X-API-Key to the control plane, never in a path, body, or envd request", async () => {
    const key = "fake-sandbox-key-SENTINEL-0001";
    const mock = await e2bServer({ apiKey: key });
    const sandbox = await activate(
      e2b(mock.url, { credential: async () => key }),
    );
    await run(sandbox, "true");
    await sandbox.dispose();
    expect(leaks(mock.requests, key, ["x-api-key"])).toEqual([]);
    for (const request of mock.requests)
      if (request.headers["x-api-key"])
        expect(request.path.startsWith("/sandboxes")).toBe(true);
  });

  it("reports a rejected key and never repeats the create; the echoed key stays out of the error", async () => {
    const key = "fake-sandbox-key-SENTINEL-0002";
    const mock = await e2bServer({ apiKey: "fake-other-key-0000" });
    let rejected = 0;
    const error = await activate(
      e2b(mock.url, {
        credential: async () => key,
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    ).catch((caught: unknown) => caught);
    expect(error).toMatchObject({ code: "SANDBOX_UNAVAILABLE" });
    expect(String((error as Error).message)).not.toContain(key);
    expect(rejected).toBe(1);
    expect(
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === "/sandboxes",
      ),
    ).toHaveLength(1);
  });

  it("repeats a renewal once with a renewed key after a rejection", async () => {
    const mock = await e2bServer({
      apiKey: "fake-sandbox-key-SENTINEL-0003",
      rejectOnce: ["/sandboxes/sbx1/timeout"],
    });
    let rejected = 0;
    const sandbox = await activate(
      e2b(mock.url, {
        credential: async () => "fake-sandbox-key-SENTINEL-0003",
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    );
    expect((await run(sandbox, "true")).exitCode).toBe(0);
    await sandbox.dispose();
    expect(rejected).toBe(1);
    expect(
      mock.requests.filter(
        (request) => request.path === "/sandboxes/sbx1/timeout",
      ).length,
    ).toBeGreaterThanOrEqual(2);
  });

  it("does not repeat a rejected request when the credential cannot be renewed", async () => {
    const mock = await e2bServer({
      apiKey: "fake-sandbox-key-SENTINEL-0004",
      rejectOnce: ["/sandboxes/sbx1/timeout"],
    });
    await expect(
      activate(
        e2b(mock.url, {
          credential: async () => "fake-sandbox-key-SENTINEL-0004",
          credentialRejected: async () => false,
        }),
      ),
    ).rejects.toThrow(/HTTP 401/);
    expect(
      mock.requests.filter(
        (request) => request.path === "/sandboxes/sbx1/timeout",
      ),
    ).toHaveLength(1);
  });

  it("does not treat a 403 from the control plane as a rejected key", async () => {
    const key = "fake-sandbox-key-SENTINEL-0005";
    const mock = await e2bServer({ apiKey: key, create: 403 });
    let rejected = 0;
    const error = await activate(
      e2b(mock.url, {
        credential: async () => key,
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    ).catch((caught: unknown) => caught);
    expect(error).toMatchObject({ code: "SANDBOX_UNAVAILABLE" });
    expect(String((error as Error).message)).toMatch(/HTTP 403/);
    expect(rejected).toBe(0);
    expect(
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === "/sandboxes",
      ),
    ).toHaveLength(1);
  });

  it("kills the remote process on timeout and reports PiShip's outcome", async () => {
    const mock = await e2bServer({
      command: (start) =>
        start.process.args[2]?.startsWith("sleep")
          ? { stdout: "working\n", hang: true }
          : { exitCode: 0 },
    });
    const sandbox = await activate(e2b(mock.url));
    await expect(run(sandbox, "sleep 600", { timeout: 0.3 })).rejects.toThrow(
      "timeout:0.3",
    );
    // The kill is sent right after PiShip stops waiting; wait for it.
    const find = () =>
      mock.requests.find(
        (request) => request.path === "/process.Process/SendSignal",
      );
    for (let attempt = 0; attempt < 100 && !find(); attempt++)
      await new Promise((done) => setTimeout(done, 20));
    const signal = find();
    const tag = (starts(mock.requests).at(-1) as StartRequest & { tag: string })
      .tag;
    expect(tag).toMatch(/^piship-[0-9a-f]{16}$/);
    expect(JSON.parse(signal?.body.toString() ?? "{}")).toEqual({
      process: { tag },
      signal: "SIGNAL_SIGKILL",
    });
    expect(signal?.headers["content-type"]).toBe("application/json");
    // The sandbox settled the cancelled stream, so it stays usable.
    expect((await run(sandbox, "true")).exitCode).toBe(0);
    await sandbox.dispose();
  });

  it("maps envd's -1 for a signalled process to a signal exit", async () => {
    const mock = await e2bServer({ command: () => ({ exitCode: -1 }) });
    const sandbox = await activate(e2b(mock.url));
    expect((await run(sandbox, "kill -9 $$")).exitCode).toBe(1);
    await sandbox.dispose();
  });

  it("runs commands as the E2B default user", async () => {
    const mock = await e2bServer();
    const sandbox = await activate(e2b(mock.url));
    await run(sandbox, "whoami");
    await sandbox.dispose();
    for (const request of mock.requests.filter(
      (item) => item.path === "/process.Process/Start",
    ))
      expect(request.headers.authorization).toBe(
        `Basic ${Buffer.from("user:").toString("base64")}`,
      );
  });

  it("runs commands as root for a CubeSandbox-style service when configured", async () => {
    const cube = await e2bServer({ user: "root" });
    // Without the setting, the service refuses the E2B default user.
    await expect(activate(e2b(cube.url))).rejects.toMatchObject({
      code: "SANDBOX_UNAVAILABLE",
      message: expect.stringContaining("HTTP 401"),
    });
    const sandbox = await activate(e2b(cube.url, { user: "root" }));
    expect((await run(sandbox, "whoami")).exitCode).toBe(0);
    await sandbox.dispose();
    const starts = cube.requests.filter(
      (item) => item.path === "/process.Process/Start",
    );
    expect(starts.at(-1)?.headers.authorization).toBe(
      `Basic ${Buffer.from("root:").toString("base64")}`,
    );
  });

  it("surfaces a failed command stream", async () => {
    const mock = await e2bServer({
      command: () => ({ streamError: "process exploded" }),
    });
    const sandbox = await activate(e2b(mock.url));
    await expect(run(sandbox, "boom")).rejects.toThrow(/process exploded/);
    await sandbox.dispose();
  });

  it("fails closed when the service is unhealthy or cannot create a sandbox", async () => {
    const unhealthy = await e2bServer({ health: 503 });
    await expect(activate(e2b(unhealthy.url))).rejects.toMatchObject({
      code: "SANDBOX_UNAVAILABLE",
      message: expect.stringContaining("HTTP 503"),
    });
    expect(
      unhealthy.requests.some((request) => request.path === "/sandboxes"),
    ).toBe(false);
    const full = await e2bServer({ create: 500 });
    await expect(activate(e2b(full.url))).rejects.toMatchObject({
      code: "SANDBOX_UNAVAILABLE",
      message: expect.stringContaining("no capacity"),
    });
  });

  it("splits Connect envelopes across chunk boundaries", () => {
    const bytes = Buffer.concat([
      connectEnvelope({ event: { start: { pid: 1 } } }),
      connectEnvelope({}, 0x02),
    ]);
    const reader = new EnvelopeReader();
    const frames = [
      ...reader.push(bytes.subarray(0, 3)),
      ...reader.push(bytes.subarray(3, 20)),
      ...reader.push(bytes.subarray(20)),
    ];
    expect(frames).toEqual([
      { flags: 0, message: { event: { start: { pid: 1 } } } },
      { flags: 2, message: {} },
    ]);
    const oversized = Buffer.alloc(5);
    oversized.writeUInt32BE(64 * 1024 * 1024, 1);
    expect(() => new EnvelopeReader().push(oversized)).toThrow(/oversized/);
  });
});

// ------------------------------------------------ Kubernetes Agent Sandbox

function kubernetes(
  url: string,
  options: Partial<KubernetesAgentSandboxOptions> = {},
) {
  return new KubernetesAgentSandboxBackend({
    endpoint: url,
    router: url,
    fetch,
    template: "python-pool",
    namespace: "agents",
    pollMs: 10,
    ...options,
  });
}

describe("kubernetes-agent-sandbox backend against a mock cluster", () => {
  it("claims a sandbox, runs through the router, and deletes the claim", async () => {
    const mock = await kubernetesServer(() => ({
      stdout: "hello\n",
      exit_code: 4,
    }));
    const sandbox = await activate(kubernetes(mock.url));
    expect(sandbox.report).toMatchObject({
      level: "enforced",
      provider: "kubernetes-agent-sandbox",
      verification: "backend-attested",
    });
    expect(
      await run(sandbox, "make test", { cwd: join(workspace, "sub") }),
    ).toEqual({ exitCode: 4, output: "hello\n" });
    await sandbox.dispose();
    const claim = JSON.parse(
      mock.requests
        .find((request) => request.method === "POST" && request.path === CLAIMS)
        ?.body.toString() ?? "{}",
    );
    expect(claim).toMatchObject({
      apiVersion: "extensions.agents.x-k8s.io/v1beta1",
      kind: "SandboxClaim",
      spec: { warmPoolRef: { name: "python-pool" } },
    });
    expect(claim.metadata.name).toMatch(/^piship-[0-9a-f]{12}$/);
    const execute = mock.requests.filter(
      (request) => request.path === "/execute",
    );
    expect(execute.at(-1)?.headers).toMatchObject({
      "x-sandbox-id": `pool-${claim.metadata.name}`,
      "x-sandbox-namespace": "agents",
      "x-sandbox-port": "8888",
    });
    expect(JSON.parse(execute.at(-1)?.body.toString() ?? "{}").command).toBe(
      `env 'LANG=C.UTF-8' 'DOCS_MODE=demo' /bin/sh -c 'cd -- '"'"'sub'"'"' && make test'`,
    );
    expect(mock.requests.at(-1)).toMatchObject({
      method: "DELETE",
      path: `${CLAIMS}/${claim.metadata.name}`,
    });
    const wire = JSON.stringify(
      mock.requests.map((request) => ({
        headers: request.headers,
        body: request.body.toString(),
      })),
    );
    for (const value of [...Object.values(SECRETS), "unlisted-host-value"])
      expect(wire).not.toContain(value);
  });

  it("replaces the claim after a timed-out command", async () => {
    const mock = await kubernetesServer((command) =>
      command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
    );
    const sandbox = await activate(kubernetes(mock.url));
    await expect(run(sandbox, "sleep 600", { timeout: 0.3 })).rejects.toThrow(
      "timeout:0.3",
    );
    expect((await run(sandbox, "echo ok")).output).toBe("ok\n");
    await sandbox.dispose();
    const created = mock.requests.filter(
      (request) => request.method === "POST" && request.path === CLAIMS,
    );
    const deleted = mock.requests.filter(
      (request) => request.method === "DELETE",
    );
    expect(created).toHaveLength(2);
    expect(deleted).toHaveLength(2);
  });

  it("shares one claim between concurrent commands and keeps it for the others when one is cancelled", async () => {
    let release!: () => void;
    const gate = new Promise<void>((done) => {
      release = done;
    });
    const mock = await kubernetesServer((command) =>
      command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
    );
    const sandbox = await activate(kubernetes(mock.url));
    const claims = () =>
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === CLAIMS,
      ).length;
    const deletes = () =>
      mock.requests.filter((request) => request.method === "DELETE").length;
    const controller = new AbortController();
    const slow = sandbox.exec("sleep 600", workspace, {
      onData: () => {},
      signal: controller.signal,
    });
    const quick = Promise.all([run(sandbox, "echo a"), run(sandbox, "echo b")]);
    expect((await quick).map((result) => result.output)).toEqual([
      "ok\n",
      "ok\n",
    ]);
    controller.abort();
    await expect(slow).rejects.toThrow("aborted");
    release();
    await gate;
    expect(claims()).toBe(1);
    // The cancelled command's claim is deleted once nothing uses it.
    expect(deletes()).toBe(1);
    const after = await Promise.all([
      run(sandbox, "echo c"),
      run(sandbox, "echo d"),
    ]);
    expect(after.map((result) => result.output)).toEqual(["ok\n", "ok\n"]);
    expect(claims()).toBe(2);
    await sandbox.dispose();
    expect(deletes()).toBe(2);
  });

  it("does not fail a parallel command that waits for the claim a cancelled one started (#158)", async () => {
    const mock = await kubernetesServer((command) =>
      command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
    );
    // Readiness takes one 300 ms poll, so both commands wait on the claim.
    const sandbox = await activate(kubernetes(mock.url, { pollMs: 300 }));
    // A cancelled command retires the first claim: the next ones share a new one.
    const first = new AbortController();
    const retired = sandbox.exec("sleep 600", workspace, {
      onData: () => {},
      signal: first.signal,
    });
    setTimeout(() => first.abort(), 50);
    await expect(retired).rejects.toThrow("aborted");
    const a = new AbortController();
    const cancelled = sandbox.exec("echo a", workspace, {
      onData: () => {},
      signal: a.signal,
    });
    const parallel = run(sandbox, "echo b");
    setTimeout(() => a.abort(), 50);
    await expect(cancelled).rejects.toThrow("aborted");
    expect((await parallel).output).toBe("ok\n");
    await sandbox.dispose();
  });

  it("keeps renewing a claim a parallel command still runs in after another is cancelled (#158)", async () => {
    let finish!: () => void;
    const done = new Promise<void>((resolve) => {
      finish = resolve;
    });
    const mock = await kubernetesServer((command) =>
      command.includes("sleep")
        ? { hang: true }
        : command.includes("long")
          ? { stdout: "long\n", after: done }
          : { stdout: "ok\n" },
    );
    // A one-second lifetime renews a running command every 250 ms.
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 1 }),
    );
    const [first] = claimNames(mock.requests);
    const renewals = () =>
      mock.requests.filter(
        (request) =>
          request.method === "PATCH" && request.path === `${CLAIMS}/${first}`,
      ).length;
    const controller = new AbortController();
    const cancelled = sandbox.exec("sleep 600", workspace, {
      onData: () => {},
      signal: controller.signal,
    });
    const parallel = run(sandbox, "long job");
    await new Promise((resolve) => setTimeout(resolve, 100));
    controller.abort();
    await expect(cancelled).rejects.toThrow("aborted");
    const atCancel = renewals();
    // The parallel command still runs in the claim: it must not expire under it.
    await new Promise((resolve) => setTimeout(resolve, 700));
    expect(renewals()).toBeGreaterThan(atCancel);
    finish();
    expect((await parallel).output).toBe("long\n");
    // Retired by the cancellation: deleted once the parallel command ends.
    expect(
      mock.requests.some(
        (request) =>
          request.method === "DELETE" && request.path === `${CLAIMS}/${first}`,
      ),
    ).toBe(true);
    await sandbox.dispose();
  });

  const claimNames = (requests: Recorded[]) =>
    requests
      .filter((request) => request.method === "POST" && request.path === CLAIMS)
      .map(
        (request) =>
          (
            JSON.parse(request.body.toString()) as {
              metadata: { name: string };
            }
          ).metadata.name,
      );
  const lifecycleOf = (request: Recorded | undefined) =>
    (
      JSON.parse(request?.body.toString() ?? "{}") as {
        spec?: {
          lifecycle?: { shutdownTime?: string; shutdownPolicy?: string };
        };
      }
    ).spec?.lifecycle;

  it("creates every claim with a bounded, deterministic lifecycle", async () => {
    const start = Date.parse("2026-09-28T18:00:00.250Z");
    const mock = await kubernetesServer();
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 900, now: () => start }),
    );
    await sandbox.dispose();
    const created = mock.requests.find(
      (request) => request.method === "POST" && request.path === CLAIMS,
    );
    // Rounded up to the second: never shorter than the lifetime.
    expect(lifecycleOf(created)).toEqual({
      shutdownTime: "2026-09-28T18:15:01Z",
      shutdownPolicy: "Delete",
    });
  });

  it("deletes the claim when a command is cancelled", async () => {
    const mock = await kubernetesServer((command) =>
      command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
    );
    const sandbox = await activate(kubernetes(mock.url));
    const [first] = claimNames(mock.requests);
    const controller = new AbortController();
    setTimeout(() => controller.abort(), 100);
    await expect(
      sandbox.exec("sleep 600", workspace, {
        onData: () => {},
        signal: controller.signal,
      }),
    ).rejects.toThrow("aborted");
    expect(
      mock.requests
        .filter((request) => request.method === "DELETE")
        .map((request) => request.path),
    ).toEqual([`${CLAIMS}/${first}`]);
    await sandbox.dispose();
  });

  it("stays fail closed when cleanup fails, and the claim still expires on its own", async () => {
    const cluster: Cluster = { deleteStatus: 500 };
    const mock = await kubernetesServer(
      (command) =>
        command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
      cluster,
    );
    const now = Date.parse("2026-09-28T18:00:00Z");
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 600, now: () => now }),
    );
    await expect(run(sandbox, "sleep 600", { timeout: 0.2 })).rejects.toThrow(
      "timeout:0.2",
    );
    const [retired] = claimNames(mock.requests);
    // The DELETE was attempted and failed; nothing about it leaked back.
    expect(
      mock.requests.some(
        (request) =>
          request.method === "DELETE" &&
          request.path === `${CLAIMS}/${retired}`,
      ),
    ).toBe(true);
    // The retired claim carries the safety net and is never renewed again.
    expect(
      lifecycleOf(
        mock.requests.find(
          (request) => request.method === "POST" && request.path === CLAIMS,
        ),
      ),
    ).toEqual({
      shutdownTime: "2026-09-28T18:10:00Z",
      shutdownPolicy: "Delete",
    });
    // The next command runs in a fresh claim, never the retired one.
    expect((await run(sandbox, "echo ok")).output).toBe("ok\n");
    const names = claimNames(mock.requests);
    expect(names).toHaveLength(2);
    const execute = mock.requests.filter(
      (request) => request.path === "/execute",
    );
    expect(execute.at(-1)?.headers["x-sandbox-id"]).toBe(`pool-${names[1]}`);
    await sandbox.dispose();
  });

  it("renews the claim so a long session is not expired early", async () => {
    let now = Date.parse("2026-09-28T18:00:00Z");
    const mock = await kubernetesServer((command) =>
      command.includes("slow") ? { hang: true } : { stdout: "ok\n" },
    );
    const sandbox = await activate(
      kubernetes(mock.url, {
        lifetimeSeconds: 1,
        now: () => now,
        monotonic: () => now,
      }),
    );
    const patches = () =>
      mock.requests.filter((request) => request.method === "PATCH");
    // Fresh claim: nothing to renew yet.
    await run(sandbox, "echo a");
    expect(patches()).toHaveLength(0);
    // Past half the lifetime, the next command renews first.
    now += 600;
    await run(sandbox, "echo b");
    expect(patches()).toHaveLength(1);
    expect(lifecycleOf(patches()[0])).toEqual({
      shutdownTime: "2026-09-28T18:00:02Z",
    });
    expect(patches()[0]?.headers["content-type"]).toBe(
      "application/merge-patch+json",
    );
    // A command that runs longer than the lifetime is kept alive while it runs.
    const controller = new AbortController();
    const slow = sandbox
      .exec("slow job", workspace, {
        onData: () => {},
        signal: controller.signal,
      })
      .catch(() => undefined);
    // With a one-second lifetime the claim is renewed every 250 ms while
    // the command runs; wait for two renewals beyond the one above.
    const deadline = Date.now() + 10_000;
    while (patches().length < 3 && Date.now() < deadline)
      await new Promise((done) => setTimeout(done, 50));
    expect(patches().length).toBeGreaterThanOrEqual(3);
    controller.abort();
    await slow;
    await sandbox.dispose();
  });

  it("stops renewing a claim once its command is cancelled", async () => {
    const mock = await kubernetesServer((command) =>
      command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
    );
    // A one-second lifetime renews a running command every 250 ms.
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 1 }),
    );
    const [first] = claimNames(mock.requests);
    const renewals = () =>
      mock.requests.filter(
        (request) =>
          request.method === "PATCH" && request.path === `${CLAIMS}/${first}`,
      ).length;
    const controller = new AbortController();
    const pending = sandbox
      .exec("sleep 600", workspace, {
        onData: () => {},
        signal: controller.signal,
      })
      .catch(() => undefined);
    // Renewal really runs while the command does...
    const deadline = Date.now() + 10_000;
    while (renewals() < 1 && Date.now() < deadline)
      await new Promise((done) => setTimeout(done, 25));
    expect(renewals()).toBeGreaterThanOrEqual(1);
    controller.abort();
    await pending;
    const atRetire = renewals();
    // ...and never again after retirement, over more than two intervals.
    await new Promise((done) => setTimeout(done, 700));
    expect(renewals()).toBe(atRetire);
    await sandbox.dispose();
  });

  it("replaces a locally expired claim without renewing it", async () => {
    let now = Date.parse("2026-09-28T18:00:00Z");
    // The cluster has not removed the claim yet, so a PATCH would succeed.
    const mock = await kubernetesServer();
    const sandbox = await activate(
      kubernetes(mock.url, {
        lifetimeSeconds: 60,
        now: () => now,
        monotonic: () => now,
      }),
    );
    const [first] = claimNames(mock.requests);
    now += 61_000;
    expect((await run(sandbox, "echo ok")).exitCode).toBe(0);
    expect(claimNames(mock.requests)).toHaveLength(2);
    expect(
      mock.requests.filter((request) => request.method === "PATCH"),
    ).toHaveLength(0);
    expect(
      mock.requests.some(
        (request) =>
          request.method === "DELETE" && request.path === `${CLAIMS}/${first}`,
      ),
    ).toBe(true);
    await sandbox.dispose();
  });

  it("replaces a claim removed early by the cluster before a command", async () => {
    const cluster: Cluster = { expired: new Set() };
    const mock = await kubernetesServer(undefined, cluster);
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 3600 }),
    );
    const [first] = claimNames(mock.requests);
    cluster.expired?.add(first ?? "");
    expect((await run(sandbox, "echo fresh")).exitCode).toBe(0);
    expect(claimNames(mock.requests)).toHaveLength(2);
    await sandbox.dispose();
  });

  it("retires a claim the router no longer knows and never replays the command", async () => {
    const cluster: Cluster = { routerGone: new Set() };
    const mock = await kubernetesServer(
      (command) => ({ stdout: `${command.split("'").at(-2)}\n` }),
      cluster,
    );
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 3600 }),
    );
    const [first] = claimNames(mock.requests);
    // The API still lists the claim, far from renewal; only the router
    // answers that its sandbox is gone.
    cluster.routerGone?.add(first ?? "");
    await expect(run(sandbox, "echo one")).rejects.toThrow(
      /no longer knows this sandbox/,
    );
    expect(
      mock.requests.some(
        (request) =>
          request.method === "DELETE" && request.path === `${CLAIMS}/${first}`,
      ),
    ).toBe(true);
    expect((await run(sandbox, "echo two")).output).toBe("echo two\n");
    const names = claimNames(mock.requests);
    expect(names).toHaveLength(2);
    const commands = mock.requests
      .filter((request) => request.path === "/execute")
      .map((request) => ({
        sandbox: request.headers["x-sandbox-id"],
        one: request.body.toString().includes("echo one"),
        two: request.body.toString().includes("echo two"),
      }))
      .filter((command) => command.one || command.two);
    // Sent once to the stale sandbox, never again; the next one is fresh.
    expect(commands).toEqual([
      { sandbox: `pool-${first}`, one: true, two: false },
      { sandbox: `pool-${names[1]}`, one: false, two: true },
    ]);
    await sandbox.dispose();
  });

  describe("a command whose outcome is unknown", () => {
    const request = {
      command: "deploy",
      cwd: workspace,
      workspacePath: ".",
      env: {},
    };
    const io = () => ({
      signal: new AbortController().signal,
      onStdout: () => {},
      onStderr: () => {},
    });

    it.each([
      ["the router connection drops", { drop: true }],
      ["the router answers 502", { status: 502 }],
      ["the router answers 504", { status: 504 }],
    ])(
      "is never retryable when %s, and its sandbox is deleted",
      async (_case, answer) => {
        const mock = await kubernetesServer((command) =>
          command.includes("deploy") ? answer : { stdout: "ok\n" },
        );
        const instance = await kubernetes(mock.url).prepare({
          profile: {} as SandboxProfile,
        });
        const error = await instance.exec(request, io()).then(
          () => undefined,
          (failure: unknown) => failure,
        );
        expect(error).toBeInstanceOf(PiShipError);
        expect(error).toMatchObject({
          code: "SANDBOX_UNAVAILABLE",
          retryable: false,
          message: expect.stringContaining("outcome is unknown"),
        });
        // The claim is deleted before the failure is reported: a command still
        // running in its pod ends with it.
        const [first] = claimNames(mock.requests);
        expect(
          mock.requests.some(
            (recorded) =>
              recorded.method === "DELETE" &&
              recorded.path === `${CLAIMS}/${first}`,
          ),
        ).toBe(true);
        // Never sent again; the next command runs in a fresh claim.
        await instance.exec({ ...request, command: "echo next" }, io());
        const names = claimNames(mock.requests);
        expect(names).toHaveLength(2);
        const execute = mock.requests.filter(
          (recorded) => recorded.path === "/execute",
        );
        expect(
          execute.filter((recorded) =>
            recorded.body.toString().includes("deploy"),
          ),
        ).toHaveLength(1);
        expect(execute.at(-1)?.headers["x-sandbox-id"]).toBe(
          `pool-${names[1]}`,
        );
        await instance.dispose();
      },
    );

    it("tells the agent not to run it again blindly", async () => {
      const mock = await kubernetesServer((command) =>
        command.includes("deploy") ? { drop: true } : { stdout: "ok\n" },
      );
      const sandbox = await activate(kubernetes(mock.url));
      const error = await run(sandbox, "deploy").then(
        () => undefined,
        (failure: unknown) => failure as Error,
      );
      expect(error?.message).toMatch(/outcome is unknown/);
      expect(error?.message).toMatch(/before running it again/);
      expect(error?.message).not.toMatch(/GATEWAY_UNREACHABLE/);
      expect((await run(sandbox, "echo ok")).output).toBe("ok\n");
      await sandbox.dispose();
    });

    it("still reports a cancelled command as cancelled", async () => {
      const mock = await kubernetesServer((command) =>
        command.includes("deploy") ? { hang: true } : { stdout: "ok\n" },
      );
      const instance = await kubernetes(mock.url).prepare({
        profile: {} as SandboxProfile,
      });
      const controller = new AbortController();
      const pending = instance.exec(request, {
        ...io(),
        signal: controller.signal,
      });
      while (
        !mock.requests.some((recorded) =>
          recorded.body.toString().includes("deploy"),
        )
      )
        await new Promise((done) => setTimeout(done, 10));
      controller.abort();
      const error = await pending.then(
        () => undefined,
        (failure: unknown) => failure,
      );
      expect(error).not.toBeInstanceOf(PiShipError);
      expect((error as Error).name).toBe("AbortError");
      await instance.dispose();
    });
  });

  it("retires a claim that disappears while a command runs, without replaying it", async () => {
    let finish!: () => void;
    const finished = new Promise<void>((done) => {
      finish = done;
    });
    const cluster: Cluster = { expired: new Set() };
    const mock = await kubernetesServer(
      (command) =>
        command.includes("slow")
          ? { stdout: "slow done\n", after: finished }
          : { stdout: "ok\n" },
      cluster,
    );
    // A one-second lifetime renews a running command every 250 ms.
    const sandbox = await activate(
      kubernetes(mock.url, { lifetimeSeconds: 1 }),
    );
    const [first] = claimNames(mock.requests);
    const slow = run(sandbox, "slow job");
    const sent = Date.now() + 10_000;
    while (
      !mock.requests.some((request) =>
        request.body.toString().includes("slow job"),
      ) &&
      Date.now() < sent
    )
      await new Promise((done) => setTimeout(done, 10));
    cluster.expired?.add(first ?? "");
    // The keepalive learns that the claim is gone.
    const deadline = Date.now() + 10_000;
    while (
      !mock.requests.some(
        (request) =>
          request.method === "PATCH" && request.path === `${CLAIMS}/${first}`,
      ) &&
      Date.now() < deadline
    )
      await new Promise((done) => setTimeout(done, 25));
    finish();
    // The command's own outcome stands; it is not sent again.
    expect((await slow).output).toBe("slow done\n");
    expect(
      mock.requests.some(
        (request) =>
          request.method === "DELETE" && request.path === `${CLAIMS}/${first}`,
      ),
    ).toBe(true);
    expect((await run(sandbox, "echo ok")).output).toBe("ok\n");
    const names = claimNames(mock.requests);
    expect(names).toHaveLength(2);
    const execute = mock.requests.filter(
      (request) => request.path === "/execute",
    );
    expect(
      execute.filter((request) => request.body.toString().includes("slow job")),
    ).toHaveLength(1);
    expect(execute.at(-1)?.headers["x-sandbox-id"]).toBe(`pool-${names[1]}`);
    await sandbox.dispose();
  });

  it("reports a new epoch once an expired claim is replaced", async () => {
    let now = Date.parse("2026-09-28T18:00:00Z");
    const mock = await kubernetesServer();
    const instance = await kubernetes(mock.url, {
      lifetimeSeconds: 60,
      now: () => now,
      monotonic: () => now,
    }).prepare({ profile: {} as SandboxProfile });
    const io = {
      signal: new AbortController().signal,
      onStdout: () => {},
      onStderr: () => {},
    };
    const request = {
      command: "true",
      cwd: workspace,
      workspacePath: ".",
      env: {},
    };
    expect(instance.epoch?.()).toBeUndefined();
    await instance.exec(request, io);
    const first = instance.epoch?.();
    expect(first).toBe(claimNames(mock.requests)[0]);
    await instance.exec(request, io);
    expect(instance.epoch?.()).toBe(first);
    now += 61_000;
    await instance.exec(request, io);
    expect(instance.epoch?.()).toBe(claimNames(mock.requests)[1]);
    expect(instance.epoch?.()).not.toBe(first);
    await instance.dispose();
  });

  it("creates no claim after dispose, even mid-renewal", async () => {
    let now = Date.parse("2026-09-28T18:00:00Z");
    let open!: () => void;
    const cluster: Cluster = {
      expired: new Set(),
      patchGate: new Promise<void>((done) => {
        open = done;
      }),
    };
    const mock = await kubernetesServer(undefined, cluster);
    const sandbox = await activate(
      kubernetes(mock.url, {
        lifetimeSeconds: 60,
        now: () => now,
        monotonic: () => now,
      }),
    );
    const [first] = claimNames(mock.requests);
    // Due for renewal; the cluster answers that the claim is gone only
    // after the session has been disposed.
    now += 40_000;
    cluster.expired?.add(first ?? "");
    const pending = run(sandbox, "echo ok");
    const deadline = Date.now() + 10_000;
    while (
      !mock.requests.some((request) => request.method === "PATCH") &&
      Date.now() < deadline
    )
      await new Promise((done) => setTimeout(done, 10));
    await sandbox.dispose();
    open();
    await expect(pending).rejects.toThrow(/disposed/);
    expect(claimNames(mock.requests)).toEqual([first]);
    expect(
      mock.requests.filter((request) => request.path === "/execute").length,
    ).toBe(1); // the activation check only
  });

  it("replaces a claim the cluster already expired before running", async () => {
    let now = Date.parse("2026-09-28T18:00:00Z");
    const cluster: Cluster = { expired: new Set() };
    const mock = await kubernetesServer(undefined, cluster);
    const sandbox = await activate(
      kubernetes(mock.url, {
        lifetimeSeconds: 60,
        now: () => now,
        monotonic: () => now,
      }),
    );
    const [first] = claimNames(mock.requests);
    cluster.expired?.add(first ?? "");
    now += 120_000;
    expect((await run(sandbox, "echo ok")).exitCode).toBe(0);
    const names = claimNames(mock.requests);
    expect(names).toHaveLength(2);
    expect(
      mock.requests.filter((request) => request.path === "/execute").at(-1)
        ?.headers["x-sandbox-id"],
    ).toBe(`pool-${names[1]}`);
    await sandbox.dispose();
  });

  describe("when the local wall clock changes", () => {
    const T = Date.parse("2026-09-28T18:00:00Z");
    const shutdowns = (requests: readonly Recorded[]) =>
      requests
        .filter(
          (request) =>
            (request.method === "POST" && request.path === CLAIMS) ||
            request.method === "PATCH",
        )
        .map((request) => lifecycleOf(request)?.shutdownTime);
    const at = (ms: number) => new Date(ms).toISOString().replace(".000Z", "Z");

    it("keeps a keepalive renewal at or after the committed shutdownTime when the clock moves backward", async () => {
      let now = T;
      // Frozen: no monotonic time passes while the clock moves.
      const monotonic = 0;
      const mock = await kubernetesServer((command) =>
        command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
      );
      // A one-second lifetime renews a running command every 250 ms.
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 1,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      now -= 3600_000;
      const controller = new AbortController();
      const pending = sandbox
        .exec("sleep 600", workspace, {
          onData: () => {},
          signal: controller.signal,
        })
        .catch(() => undefined);
      const deadline = Date.now() + 10_000;
      while (
        !mock.requests.some((request) => request.method === "PATCH") &&
        Date.now() < deadline
      )
        await new Promise((done) => setTimeout(done, 25));
      controller.abort();
      await pending;
      const values = shutdowns(mock.requests);
      expect(values.length).toBeGreaterThanOrEqual(2);
      // No monotonic time passed: every renewal commits the creation value.
      for (const value of values) expect(value).toBe(at(T + 1000));
      await sandbox.dispose();
    });

    it.each([
      ["backward by an hour", -3600_000],
      ["backward by two seconds (a small correction)", -2000],
    ])(
      "renews from monotonic elapsed time when the clock moves %s",
      async (_name, jumpMs) => {
        let now = T;
        let monotonic = 5000;
        const mock = await kubernetesServer();
        const sandbox = await activate(
          kubernetes(mock.url, {
            lifetimeSeconds: 60,
            now: () => now,
            monotonic: () => monotonic,
          }),
        );
        now += jumpMs;
        monotonic += 40_000;
        await run(sandbox, "echo a");
        // Several renewals, with the clock moving again between them.
        now -= jumpMs / 2;
        monotonic += 40_000;
        await run(sandbox, "echo b");
        expect(shutdowns(mock.requests)).toEqual([
          at(T + 60_000),
          at(T + 100_000),
          at(T + 140_000),
        ]);
        expect(claimNames(mock.requests)).toHaveLength(1);
        // The local deadline is the committed one: just before it the claim
        // is renewed and kept, just after it replaced without renewal.
        monotonic += 59_999;
        await run(sandbox, "echo c");
        expect(claimNames(mock.requests)).toHaveLength(1);
        // 139.999 s elapsed plus the lifetime, rounded up to the second.
        expect(shutdowns(mock.requests).at(-1)).toBe(at(T + 200_000));
        monotonic += 60_001;
        await run(sandbox, "echo d");
        expect(claimNames(mock.requests)).toHaveLength(2);
        await sandbox.dispose();
      },
    );

    it("renews on wall elapsed time after a suspend stopped the monotonic clock", async () => {
      let now = T;
      let monotonic = 0;
      const mock = await kubernetesServer();
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 3600,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      // Asleep for 45 minutes; the monotonic clock advanced 5.
      now += 45 * 60_000;
      monotonic += 5 * 60_000;
      await run(sandbox, "echo a");
      expect(shutdowns(mock.requests)).toEqual([
        at(T + 3600_000),
        at(T + 45 * 60_000 + 3600_000),
      ]);
      expect(claimNames(mock.requests)).toHaveLength(1);
      await sandbox.dispose();
    });

    it("replaces a claim the wall clock says expired during a suspend, without reviving it", async () => {
      let now = T;
      let monotonic = 0;
      const mock = await kubernetesServer();
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 3600,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      now += 61 * 60_000;
      monotonic += 5 * 60_000;
      await run(sandbox, "echo ok");
      expect(claimNames(mock.requests)).toHaveLength(2);
      expect(
        mock.requests.filter((request) => request.method === "PATCH"),
      ).toHaveLength(0);
      await sandbox.dispose();
    });

    it("never lets a late keepalive revive a claim past its shutdownTime", async () => {
      let now = T;
      // Frozen: the machine slept, only the wall clock moved.
      const monotonic = 0;
      const mock = await kubernetesServer((command) =>
        command.includes("sleep") ? { hang: true } : { stdout: "ok\n" },
      );
      // A one-second lifetime renews a running command every 250 ms.
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 1,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      const controller = new AbortController();
      const pending = sandbox
        .exec("sleep 600", workspace, {
          onData: () => {},
          signal: controller.signal,
        })
        .catch(() => undefined);
      const sent = Date.now() + 10_000;
      while (
        !mock.requests.some((request) =>
          request.body.toString().includes("sleep 600"),
        ) &&
        Date.now() < sent
      )
        await new Promise((done) => setTimeout(done, 10));
      // The command runs; the machine sleeps past the claim's shutdownTime.
      now += 5000;
      // Two keepalive intervals pass with the claim already expired.
      await new Promise((done) => setTimeout(done, 700));
      expect(
        mock.requests.filter((request) => request.method === "PATCH"),
      ).toHaveLength(0);
      controller.abort();
      await pending;
      await sandbox.dispose();
    });

    it("counts a forward jump as elapsed time: renewal comes earlier, and shutdownTime moves by at most the jump", async () => {
      let now = T;
      let monotonic = 0;
      const mock = await kubernetesServer();
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 60,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      // 10 s pass and the clock is also set 25 s ahead: 35 s count.
      now += 35_000;
      monotonic += 10_000;
      await run(sandbox, "echo a");
      // Monotonic time alone would neither renew yet nor give T + 70 s;
      // the jump lengthens the lifetime by at most its own 25 s.
      expect(shutdowns(mock.requests)).toEqual([
        at(T + 60_000),
        at(T + 95_000),
      ]);
      // A jump past the committed shutdownTime counts as expiry.
      now += 86_400_000;
      await run(sandbox, "echo b");
      expect(claimNames(mock.requests)).toHaveLength(2);
      expect(
        mock.requests.filter((request) => request.method === "PATCH"),
      ).toHaveLength(1);
      await sandbox.dispose();
    });

    it("expires a claim locally at its committed shutdownTime after the clock moved backward", async () => {
      let now = T;
      let monotonic = 0;
      const mock = await kubernetesServer();
      const sandbox = await activate(
        kubernetes(mock.url, {
          lifetimeSeconds: 60,
          now: () => now,
          monotonic: () => monotonic,
        }),
      );
      now -= 3600_000;
      monotonic += 61_000;
      await run(sandbox, "echo ok");
      // Past the committed shutdownTime: replaced, never revived by a PATCH.
      expect(claimNames(mock.requests)).toHaveLength(2);
      expect(
        mock.requests.filter((request) => request.method === "PATCH"),
      ).toHaveLength(0);
      await sandbox.dispose();
    });

    it.each([
      ["backward", -3600_000],
      ["forward", 3600_000],
    ])(
      "times out waiting for Ready after the configured time when the clock jumps %s",
      async (_direction, jumpMs) => {
        const mock = await kubernetesServer(undefined, { neverReady: true });
        const wall = Date.now;
        const started = performance.now();
        const spy = vi
          .spyOn(Date, "now")
          .mockImplementation(
            () => wall() + (performance.now() - started > 100 ? jumpMs : 0),
          );
        try {
          const failure = await kubernetes(mock.url, { readyTimeoutMs: 600 })
            .prepare({ profile: {} as SandboxProfile })
            .catch((error: unknown) => error);
          const elapsed = performance.now() - started;
          expect(String(failure)).toContain("did not become ready in time");
          // Not cut short by a forward jump; the generous upper bound only
          // has to catch a wait a backward jump stretched by the hour.
          expect(elapsed).toBeGreaterThanOrEqual(550);
          expect(elapsed).toBeLessThan(15_000);
        } finally {
          spy.mockRestore();
        }
      },
      30_000,
    );
  });

  it("disposes idempotently", async () => {
    const mock = await kubernetesServer();
    const sandbox = await activate(kubernetes(mock.url));
    await Promise.all([sandbox.dispose(), sandbox.dispose()]);
    await sandbox.dispose();
    expect(
      mock.requests.filter((request) => request.method === "DELETE"),
    ).toHaveLength(1);
  });

  it("sends a bearer to the API and the router only when a credential is given", async () => {
    const mock = await kubernetesServer();
    const sandbox = await activate(
      kubernetes(mock.url, { credential: async () => "runtime-credential-2" }),
    );
    await run(sandbox, "true");
    await sandbox.dispose();
    for (const request of mock.requests)
      expect(request.headers.authorization).toBe("Bearer runtime-credential-2");
    expect(
      leaks(mock.requests, "runtime-credential-2", ["authorization"]),
    ).toEqual([]);
  });

  it("reports an RBAC 403 on claim creation without rejecting the bearer", async () => {
    const token = "fake-bearer-SENTINEL-0006";
    const mock = await kubernetesServer(undefined, {
      token,
      createStatus: 403,
    });
    let rejected = 0;
    const error = await activate(
      kubernetes(mock.url, {
        credential: async () => token,
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    ).catch((caught: unknown) => caught);
    expect(error).toMatchObject({ code: "SANDBOX_UNAVAILABLE" });
    const message = String((error as Error).message);
    expect(message).toMatch(/creating the SandboxClaim failed: HTTP 403/);
    expect(message).not.toContain(token);
    expect(rejected).toBe(0);
    expect(
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === CLAIMS,
      ),
    ).toHaveLength(1);
  });

  it("reports an RBAC 403 from the router without rejecting the bearer", async () => {
    const token = "fake-bearer-SENTINEL-0007";
    const mock = await kubernetesServer(
      (command) =>
        command.includes("forbidden") ? { status: 403 } : { stdout: "ok\n" },
      { token },
    );
    let rejected = 0;
    const sandbox = await activate(
      kubernetes(mock.url, {
        credential: async () => token,
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    );
    const error = await run(sandbox, "echo forbidden").catch(
      (caught: unknown) => caught,
    );
    const message = String((error as Error).message);
    expect(message).toMatch(/running the command failed: HTTP 403/);
    expect(message).not.toContain(token);
    expect(rejected).toBe(0);
    // The credential stays usable for the next command.
    expect((await run(sandbox, "echo ok")).output).toBe("ok\n");
    expect(rejected).toBe(0);
    await sandbox.dispose();
  });

  it("rejects the bearer on a 401 and creates no claim", async () => {
    const mock = await kubernetesServer(undefined, {
      token: "fake-bearer-expected-0000",
    });
    let rejected = 0;
    const error = await activate(
      kubernetes(mock.url, {
        credential: async () => "fake-bearer-SENTINEL-0008",
        credentialRejected: async () => {
          rejected++;
          return true;
        },
      }),
    ).catch((caught: unknown) => caught);
    expect(error).toMatchObject({ code: "SANDBOX_UNAVAILABLE" });
    const message = String((error as Error).message);
    expect(message).toMatch(/HTTP 401/);
    expect(message).not.toContain("fake-bearer-SENTINEL-0008");
    expect(rejected).toBeGreaterThanOrEqual(1);
    // The API refused the bearer before any claim was requested.
    expect(
      mock.requests.filter(
        (request) => request.method === "POST" && request.path === CLAIMS,
      ),
    ).toHaveLength(0);
  });

  it("quotes commands and environment for the runtime's shell-like split", () => {
    expect(
      runtimeCommand(
        {
          command: 'echo "it\'s" && ls',
          cwd: "/unused",
          workspacePath: ".",
          env: { A: "x y", "BAD NAME": "z" },
        },
        undefined,
      ),
    ).toBe(`env 'A=x y' /bin/sh -c 'echo "it'"'"'s" && ls'`);
    expect(
      runtimeCommand(
        { command: "ls", cwd: "/unused", workspacePath: "a/b", env: {} },
        "/workspace",
      ),
    ).toBe(`env /bin/sh -c 'cd -- '"'"'/workspace/a/b'"'"' && ls'`);
  });

  it("rejects names that are not Kubernetes resource names", () => {
    expect(() =>
      kubernetes("http://127.0.0.1:1", { namespace: "Agents" }),
    ).toThrow(/Kubernetes resource names/);
  });
});