Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .server-changes/pod-count-exact-list.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: supervisor
type: improvement
---

Self-hosted Kubernetes deployments now pause and resume pulling new work at their exact configured thresholds, rather than slightly early or late.
72 changes: 38 additions & 34 deletions apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts
Original file line number Diff line number Diff line change
@@ -1,52 +1,56 @@
import { describe, it, expect } from "vitest";
import { parsePodCount, K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
import { K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
import { createPodCountFetcher } from "../clients/kubernetes.js";
import type { K8sApi } from "../clients/kubernetes.js";

describe("parsePodCount", () => {
it("reads the pods object count", () => {
const text = [
"# HELP apiserver_storage_objects Number of stored objects",
"# TYPE apiserver_storage_objects gauge",
'apiserver_storage_objects{resource="pods"} 8421',
'apiserver_storage_objects{resource="configmaps"} 17',
].join("\n");
expect(parsePodCount(text)).toBe(8421);
function apiReturning(list: unknown): K8sApi {
return { core: { listNamespacedPod: async () => list } } as unknown as K8sApi;
}
Comment thread
nicktrn marked this conversation as resolved.
Resolved

describe("createPodCountFetcher", () => {
it("returns items.length when the list is not truncated", async () => {
const fetch = createPodCountFetcher(
apiReturning({ items: [{}], metadata: {} }),
"v4-runs",
1000
);
expect(await fetch()).toBe(1);
});

it("is tolerant of extra labels in any order", () => {
const text = 'apiserver_storage_objects{group="",resource="pods",extra="x"} 12';
expect(parsePodCount(text)).toBe(12);
it("returns zero for an empty namespace", async () => {
const fetch = createPodCountFetcher(apiReturning({ items: [], metadata: {} }), "v4-runs", 1000);
expect(await fetch()).toBe(0);
});

it("parses scientific notation", () => {
const text = 'apiserver_storage_objects{resource="pods"} 1.2e+04';
expect(parsePodCount(text)).toBe(12000);
it("adds remainingItemCount when the list is truncated", async () => {
const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: 24492 } };
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
expect(await fetch()).toBe(24493);
});

it("throws when the pods metric is absent", () => {
const text = 'apiserver_storage_objects{resource="configmaps"} 17';
expect(() => parsePodCount(text)).toThrow(/not found/);
it("throws when truncated but remainingItemCount is absent", async () => {
const list = { items: [{}], metadata: { _continue: "tok" } };
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
await expect(fetch()).rejects.toThrow(/remainingItemCount/);
});

it("throws on a non-finite value (e.g. 1e999)", () => {
const text = 'apiserver_storage_objects{resource="pods"} 1e999';
expect(() => parsePodCount(text)).toThrow();
it("throws when truncated but remainingItemCount is negative", async () => {
const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: -1 } };
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
await expect(fetch()).rejects.toThrow(/remainingItemCount/);
});

it("throws on a negative value", () => {
const text = 'apiserver_storage_objects{resource="pods"} -5';
expect(() => parsePodCount(text)).toThrow();
it("rejects when the list hangs past the timeout", async () => {
const api = { core: { listNamespacedPod: () => new Promise(() => {}) } } as unknown as K8sApi;
await expect(createPodCountFetcher(api, "v4-runs", 10)()).rejects.toThrow(/timed out/);
});
});

function metrics(count: number): string {
return `apiserver_storage_objects{resource="pods"} ${count}`;
}

describe("K8sPodCountSignalSource", () => {
it("engages at the engage threshold and reports the count", async () => {
const counts: number[] = [];
const source = new K8sPodCountSignalSource({
fetchMetrics: async () => metrics(10000),
fetchPodCount: async () => 10000,
engageThreshold: 10000,
releaseThreshold: 5000,
reportPodCount: (c) => counts.push(c),
Expand All @@ -59,7 +63,7 @@ describe("K8sPodCountSignalSource", () => {

it("does not engage below the engage threshold", async () => {
const source = new K8sPodCountSignalSource({
fetchMetrics: async () => metrics(9999),
fetchPodCount: async () => 9999,
engageThreshold: 10000,
releaseThreshold: 5000,
});
Expand All @@ -69,7 +73,7 @@ describe("K8sPodCountSignalSource", () => {
it("stays engaged in the hysteresis band, releases only below release threshold", async () => {
let count = 10000;
const source = new K8sPodCountSignalSource({
fetchMetrics: async () => metrics(count),
fetchPodCount: async () => count,
engageThreshold: 10000,
releaseThreshold: 5000,
});
Expand All @@ -82,9 +86,9 @@ describe("K8sPodCountSignalSource", () => {
expect((await source.read()).engaged).toBe(false); // band again -> stays off
});

it("propagates scrape failures (monitor fails open on throw)", async () => {
it("propagates fetch failures (monitor fails open on throw)", async () => {
const source = new K8sPodCountSignalSource({
fetchMetrics: async () => {
fetchPodCount: async () => {
throw new Error("connection refused");
},
engageThreshold: 10000,
Expand Down
20 changes: 2 additions & 18 deletions apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,7 @@
import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js";

// Reads the apiserver's stored-pod-object count from a Prometheus /metrics scrape.
const POD_COUNT_RE = /^apiserver_storage_objects\{[^}]*resource="pods"[^}]*\}\s+([0-9.eE+]+)/m;

export function parsePodCount(metricsText: string): number {
const match = metricsText.match(POD_COUNT_RE);
if (!match) {
throw new Error('apiserver_storage_objects{resource="pods"} not found in metrics');
}
const value = Number(match[1]);
if (!Number.isFinite(value)) {
throw new Error(`unparseable pod count: ${match[1]}`);
}
return value;
}

export type K8sPodCountSignalSourceOptions = {
fetchMetrics: () => Promise<string>;
fetchPodCount: () => Promise<number>;
engageThreshold: number;
releaseThreshold: number;
reportPodCount?: (count: number) => void;
Expand All @@ -29,8 +14,7 @@ export class K8sPodCountSignalSource implements BackpressureSignalSource {
constructor(private readonly opts: K8sPodCountSignalSourceOptions) {}

async read(): Promise<BackpressureVerdict> {
const text = await this.opts.fetchMetrics();
const count = parsePodCount(text);
const count = await this.opts.fetchPodCount();
this.opts.reportPodCount?.(count);

if (this.engaged) {
Expand Down
101 changes: 54 additions & 47 deletions apps/supervisor/src/clients/kubernetes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import * as k8s from "@kubernetes/client-node";
import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node";
import { assertExhaustive } from "@trigger.dev/core/utils";
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import * as https from "node:https";

export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";

Expand Down Expand Up @@ -54,55 +53,63 @@ function getKubeConfig() {
export { k8s };

/**
* Builds a function that scrapes the apiserver's Prometheus /metrics endpoint.
* One lightweight aggregate read - not a pod listing. Requires the service
* account to be granted GET on the /metrics non-resource URL.
* createPodCountFetcher counts pod objects in a namespace with a single `limit=1`
* list: one pod transferred, no informer, no watch cache. Population is
* `remainingItemCount + items.length`.
*
* Two request-shape constraints, both load-bearing. A label or field selector makes
* the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion`
* serves a cached count instead of a quorum read - so neither is passed.
*
* `remainingItemCount` is only set when the list is truncated, so `_continue` is the
* truncation signal: absent means the returned page is the whole collection.
*/
export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> {
const kubeConfig = getKubeConfig();
export function createPodCountFetcher(
api: K8sApi,
namespace: string,
timeoutMs: number
): () => Promise<number> {
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
const serverTimeoutSeconds = Math.max(1, Math.floor(timeoutMs / 1000));

return async () => {
const cluster = kubeConfig.getCurrentCluster();
if (!cluster) {
throw new Error("no current cluster in kubeconfig");
const list = await withTimeout(
api.core.listNamespacedPod({
namespace,
limit: 1,
timeoutSeconds: serverTimeoutSeconds,
}),
timeoutMs,
"pod count list"
);

if (!list.metadata?._continue) {
return list.items.length; // not truncated, so this is all of them
}
const url = new URL(`${cluster.server}/metrics`);
const opts: https.RequestOptions = {
method: "GET",
protocol: url.protocol,
hostname: url.hostname,
port: url.port,
path: url.pathname,
};
// applyToHTTPSOptions sets the cluster CA, client cert/key, and auth headers
// (incl. exec plugins) on the request - so TLS verifies against the cluster
// CA, not the system store. The fetch-options path attaches the CA as an
// https.Agent, which global fetch (undici) ignores.
await kubeConfig.applyToHTTPSOptions(opts);

return new Promise<string>((resolve, reject) => {
const req = https.request(opts, (res) => {
const status = res.statusCode ?? 0;
let body = "";
res.setEncoding("utf8");
res.on("data", (chunk) => {
body += chunk;
});
res.on("end", () => {
if (status >= 200 && status < 300) {
resolve(body);
} else {
reject(new Error(`apiserver /metrics scrape failed: ${status}`));
}
});
});
// Without this a hung connect/TLS/read never settles, and the monitor's
// refreshInFlight guard would freeze the source (silent fail-open).
req.setTimeout(timeoutMs, () => {
req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`));
});
req.on("error", reject);
req.end();
});

const remaining = list.metadata.remainingItemCount;
if (typeof remaining !== "number" || !Number.isFinite(remaining) || remaining < 0) {
throw new Error("pod list truncated but remainingItemCount absent or invalid");
}

return list.items.length + remaining;
};
}

/**
* withTimeout rejects if `promise` outlives `timeoutMs`, so a hung request cannot
* freeze the caller. A backstop only: the k8s client threads no AbortSignal through
* to fetch, so callers must also bound the request server-side (`timeoutSeconds`),
* or an abandoned request would stay open. The timer is cleared either way.
*/
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, what: string): Promise<T> {
let timer: NodeJS.Timeout;
const deadline = new Promise<never>((_resolve, reject) => {
timer = setTimeout(
() => reject(new Error(`${what} timed out after ${timeoutMs}ms`)),
timeoutMs
);
timer.unref();
});

return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
}
Comment thread
nicktrn marked this conversation as resolved.
Resolved
8 changes: 5 additions & 3 deletions apps/supervisor/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
CheckpointClient,
isKubernetesEnvironment,
} from "@trigger.dev/core/v3/serverOnly";
import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
import { createK8sApi, createPodCountFetcher } from "./clients/kubernetes.js";
import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client";
import { register } from "./metrics.js";
import { PodCleaner } from "./services/podCleaner.js";
Expand Down Expand Up @@ -276,14 +276,16 @@ class ManagedSupervisor {
// RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
const podCountGauge = new Gauge({
name: "supervisor_cluster_pod_count",
help: "Total pod objects stored in the cluster, scraped for backpressure",
help: "Pod objects in the workload namespace, counted for backpressure",
registers: [register],
});
this.backpressureMonitors.push(
new BackpressureMonitor({
enabled: true,
source: new K8sPodCountSignalSource({
fetchMetrics: createApiserverMetricsFetcher(
fetchPodCount: createPodCountFetcher(
createK8sApi(),
env.KUBERNETES_NAMESPACE,
env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
),
engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
Comment thread
nicktrn marked this conversation as resolved.
Resolved
Expand Down