Skip to content

Commit 8cec1d1

Browse files
committed
fix(supervisor): count pods with an exact list instead of an aggregate metric
1 parent f10bc23 commit 8cec1d1

4 files changed

Lines changed: 81 additions & 103 deletions

File tree

‎apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,52 @@
11
import { describe, it, expect } from "vitest";
2-
import { parsePodCount, K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
2+
import { K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
3+
import { createPodCountFetcher } from "../clients/kubernetes.js";
4+
import type { K8sApi } from "../clients/kubernetes.js";
35

4-
describe("parsePodCount", () => {
5-
it("reads the pods object count", () => {
6-
const text = [
7-
"# HELP apiserver_storage_objects Number of stored objects",
8-
"# TYPE apiserver_storage_objects gauge",
9-
'apiserver_storage_objects{resource="pods"} 8421',
10-
'apiserver_storage_objects{resource="configmaps"} 17',
11-
].join("\n");
12-
expect(parsePodCount(text)).toBe(8421);
6+
function apiReturning(list: unknown): K8sApi {
7+
return { core: { listNamespacedPod: async () => list } } as unknown as K8sApi;
8+
}
9+
10+
describe("createPodCountFetcher", () => {
11+
it("returns items.length when the list is not truncated", async () => {
12+
const fetch = createPodCountFetcher(apiReturning({ items: [{}], metadata: {} }), "v4-runs", 1000);
13+
expect(await fetch()).toBe(1);
1314
});
1415

15-
it("is tolerant of extra labels in any order", () => {
16-
const text = 'apiserver_storage_objects{group="",resource="pods",extra="x"} 12';
17-
expect(parsePodCount(text)).toBe(12);
16+
it("returns zero for an empty namespace", async () => {
17+
const fetch = createPodCountFetcher(apiReturning({ items: [], metadata: {} }), "v4-runs", 1000);
18+
expect(await fetch()).toBe(0);
1819
});
1920

20-
it("parses scientific notation", () => {
21-
const text = 'apiserver_storage_objects{resource="pods"} 1.2e+04';
22-
expect(parsePodCount(text)).toBe(12000);
21+
it("adds remainingItemCount when the list is truncated", async () => {
22+
const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: 24492 } };
23+
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
24+
expect(await fetch()).toBe(24493);
2325
});
2426

25-
it("throws when the pods metric is absent", () => {
26-
const text = 'apiserver_storage_objects{resource="configmaps"} 17';
27-
expect(() => parsePodCount(text)).toThrow(/not found/);
27+
it("throws when truncated but remainingItemCount is absent", async () => {
28+
const list = { items: [{}], metadata: { _continue: "tok" } };
29+
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
30+
await expect(fetch()).rejects.toThrow(/remainingItemCount/);
2831
});
2932

30-
it("throws on a non-finite value (e.g. 1e999)", () => {
31-
const text = 'apiserver_storage_objects{resource="pods"} 1e999';
32-
expect(() => parsePodCount(text)).toThrow();
33+
it("throws when truncated but remainingItemCount is negative", async () => {
34+
const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: -1 } };
35+
const fetch = createPodCountFetcher(apiReturning(list), "v4-runs", 1000);
36+
await expect(fetch()).rejects.toThrow(/remainingItemCount/);
3337
});
3438

35-
it("throws on a negative value", () => {
36-
const text = 'apiserver_storage_objects{resource="pods"} -5';
37-
expect(() => parsePodCount(text)).toThrow();
39+
it("rejects when the list hangs past the timeout", async () => {
40+
const api = { core: { listNamespacedPod: () => new Promise(() => {}) } } as unknown as K8sApi;
41+
await expect(createPodCountFetcher(api, "v4-runs", 10)()).rejects.toThrow(/timed out/);
3842
});
3943
});
4044

41-
function metrics(count: number): string {
42-
return `apiserver_storage_objects{resource="pods"} ${count}`;
43-
}
44-
4545
describe("K8sPodCountSignalSource", () => {
4646
it("engages at the engage threshold and reports the count", async () => {
4747
const counts: number[] = [];
4848
const source = new K8sPodCountSignalSource({
49-
fetchMetrics: async () => metrics(10000),
49+
fetchPodCount: async () => 10000,
5050
engageThreshold: 10000,
5151
releaseThreshold: 5000,
5252
reportPodCount: (c) => counts.push(c),
@@ -59,7 +59,7 @@ describe("K8sPodCountSignalSource", () => {
5959

6060
it("does not engage below the engage threshold", async () => {
6161
const source = new K8sPodCountSignalSource({
62-
fetchMetrics: async () => metrics(9999),
62+
fetchPodCount: async () => 9999,
6363
engageThreshold: 10000,
6464
releaseThreshold: 5000,
6565
});
@@ -69,7 +69,7 @@ describe("K8sPodCountSignalSource", () => {
6969
it("stays engaged in the hysteresis band, releases only below release threshold", async () => {
7070
let count = 10000;
7171
const source = new K8sPodCountSignalSource({
72-
fetchMetrics: async () => metrics(count),
72+
fetchPodCount: async () => count,
7373
engageThreshold: 10000,
7474
releaseThreshold: 5000,
7575
});
@@ -82,9 +82,9 @@ describe("K8sPodCountSignalSource", () => {
8282
expect((await source.read()).engaged).toBe(false); // band again -> stays off
8383
});
8484

85-
it("propagates scrape failures (monitor fails open on throw)", async () => {
85+
it("propagates fetch failures (monitor fails open on throw)", async () => {
8686
const source = new K8sPodCountSignalSource({
87-
fetchMetrics: async () => {
87+
fetchPodCount: async () => {
8888
throw new Error("connection refused");
8989
},
9090
engageThreshold: 10000,

‎apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,7 @@
11
import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js";
22

3-
// Reads the apiserver's stored-pod-object count from a Prometheus /metrics scrape.
4-
const POD_COUNT_RE = /^apiserver_storage_objects\{[^}]*resource="pods"[^}]*\}\s+([0-9.eE+]+)/m;
5-
6-
export function parsePodCount(metricsText: string): number {
7-
const match = metricsText.match(POD_COUNT_RE);
8-
if (!match) {
9-
throw new Error('apiserver_storage_objects{resource="pods"} not found in metrics');
10-
}
11-
const value = Number(match[1]);
12-
if (!Number.isFinite(value)) {
13-
throw new Error(`unparseable pod count: ${match[1]}`);
14-
}
15-
return value;
16-
}
17-
183
export type K8sPodCountSignalSourceOptions = {
19-
fetchMetrics: () => Promise<string>;
4+
fetchPodCount: () => Promise<number>;
205
engageThreshold: number;
216
releaseThreshold: number;
227
reportPodCount?: (count: number) => void;
@@ -29,8 +14,7 @@ export class K8sPodCountSignalSource implements BackpressureSignalSource {
2914
constructor(private readonly opts: K8sPodCountSignalSourceOptions) {}
3015

3116
async read(): Promise<BackpressureVerdict> {
32-
const text = await this.opts.fetchMetrics();
33-
const count = parsePodCount(text);
17+
const count = await this.opts.fetchPodCount();
3418
this.opts.reportPodCount?.(count);
3519

3620
if (this.engaged) {

‎apps/supervisor/src/clients/kubernetes.ts‎

Lines changed: 40 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import * as k8s from "@kubernetes/client-node";
22
import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node";
33
import { assertExhaustive } from "@trigger.dev/core/utils";
44
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
5-
import * as https from "node:https";
65

76
export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
87

@@ -54,55 +53,48 @@ function getKubeConfig() {
5453
export { k8s };
5554

5655
/**
57-
* Builds a function that scrapes the apiserver's Prometheus /metrics endpoint.
58-
* One lightweight aggregate read - not a pod listing. Requires the service
59-
* account to be granted GET on the /metrics non-resource URL.
56+
* createPodCountFetcher counts pod objects in a namespace with a single `limit=1`
57+
* list: one pod transferred, no informer, no watch cache. Population is
58+
* `remainingItemCount + items.length`.
59+
*
60+
* Two request-shape constraints, both load-bearing. A label or field selector makes
61+
* the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion`
62+
* serves a cached count instead of a quorum read - so neither is passed.
63+
*
64+
* `remainingItemCount` is only set when the list is truncated, so `_continue` is the
65+
* truncation signal: absent means the returned page is the whole collection.
6066
*/
61-
export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> {
62-
const kubeConfig = getKubeConfig();
63-
67+
export function createPodCountFetcher(
68+
api: K8sApi,
69+
namespace: string,
70+
timeoutMs: number
71+
): () => Promise<number> {
6472
return async () => {
65-
const cluster = kubeConfig.getCurrentCluster();
66-
if (!cluster) {
67-
throw new Error("no current cluster in kubeconfig");
73+
const list = await withTimeout(
74+
api.core.listNamespacedPod({ namespace, limit: 1 }),
75+
timeoutMs,
76+
"pod count list"
77+
);
78+
79+
if (!list.metadata?._continue) {
80+
return list.items.length; // not truncated, so this is all of them
81+
}
82+
83+
const remaining = list.metadata.remainingItemCount;
84+
if (typeof remaining !== "number" || !Number.isFinite(remaining) || remaining < 0) {
85+
throw new Error("pod list truncated but remainingItemCount absent or invalid");
6886
}
69-
const url = new URL(`${cluster.server}/metrics`);
70-
const opts: https.RequestOptions = {
71-
method: "GET",
72-
protocol: url.protocol,
73-
hostname: url.hostname,
74-
port: url.port,
75-
path: url.pathname,
76-
};
77-
// applyToHTTPSOptions sets the cluster CA, client cert/key, and auth headers
78-
// (incl. exec plugins) on the request - so TLS verifies against the cluster
79-
// CA, not the system store. The fetch-options path attaches the CA as an
80-
// https.Agent, which global fetch (undici) ignores.
81-
await kubeConfig.applyToHTTPSOptions(opts);
82-
83-
return new Promise<string>((resolve, reject) => {
84-
const req = https.request(opts, (res) => {
85-
const status = res.statusCode ?? 0;
86-
let body = "";
87-
res.setEncoding("utf8");
88-
res.on("data", (chunk) => {
89-
body += chunk;
90-
});
91-
res.on("end", () => {
92-
if (status >= 200 && status < 300) {
93-
resolve(body);
94-
} else {
95-
reject(new Error(`apiserver /metrics scrape failed: ${status}`));
96-
}
97-
});
98-
});
99-
// Without this a hung connect/TLS/read never settles, and the monitor's
100-
// refreshInFlight guard would freeze the source (silent fail-open).
101-
req.setTimeout(timeoutMs, () => {
102-
req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`));
103-
});
104-
req.on("error", reject);
105-
req.end();
106-
});
87+
88+
return list.items.length + remaining;
10789
};
10890
}
91+
92+
/** Rejects if `promise` outlives `timeoutMs`, so a hung request cannot freeze the caller. */
93+
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, what: string): Promise<T> {
94+
return Promise.race([
95+
promise,
96+
new Promise<never>((_resolve, reject) => {
97+
setTimeout(() => reject(new Error(`${what} timed out after ${timeoutMs}ms`)), timeoutMs).unref();
98+
}),
99+
]);
100+
}

‎apps/supervisor/src/index.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
CheckpointClient,
2222
isKubernetesEnvironment,
2323
} from "@trigger.dev/core/v3/serverOnly";
24-
import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
24+
import { createK8sApi, createPodCountFetcher } from "./clients/kubernetes.js";
2525
import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client";
2626
import { register } from "./metrics.js";
2727
import { PodCleaner } from "./services/podCleaner.js";
@@ -276,14 +276,16 @@ class ManagedSupervisor {
276276
// RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
277277
const podCountGauge = new Gauge({
278278
name: "supervisor_cluster_pod_count",
279-
help: "Total pod objects stored in the cluster, scraped for backpressure",
279+
help: "Pod objects in the workload namespace, counted for backpressure",
280280
registers: [register],
281281
});
282282
this.backpressureMonitors.push(
283283
new BackpressureMonitor({
284284
enabled: true,
285285
source: new K8sPodCountSignalSource({
286-
fetchMetrics: createApiserverMetricsFetcher(
286+
fetchPodCount: createPodCountFetcher(
287+
createK8sApi(),
288+
env.KUBERNETES_NAMESPACE,
287289
env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
288290
),
289291
engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,

0 commit comments

Comments
 (0)