Skip to content

Commit b8a873e

Browse files
duizabojulclaude
andcommitted
fix: use inactivity timeout for watch instead of whole-request timeout
Watch.watch() passed AbortSignal.timeout(requestTimeoutMs) to fetch, which counts from request start and stays armed while the response body streams. Any watch, healthy or not, was therefore aborted with a TimeoutError after 30s. Replace it with a resettable timer on the AbortController: it is armed before the fetch (connect timeout), then re-armed on the 200 response and on every body chunk, so requestTimeoutMs now means "max time without data". The timer is unref'd and cleared when the watch is done. No public API change. Also reset ListWatch's reconnectDelayMs to 0 in the TimeoutError branch of doneHandler. A client-side timeout means we already waited the full request timeout, so reconnecting immediately cannot tight-loop, while backing off would leave quiet resources unwatched for up to MAX_RECONNECT_DELAY_MS. Exponential backoff still applies to real errors and server-side disconnects. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012eAdChkTVkV5KHSedtsML4
1 parent d8ac4ac commit b8a873e

4 files changed

Lines changed: 173 additions & 6 deletions

File tree

‎src/cache.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,11 @@ export class ListWatch<T extends KubernetesObject> implements ObjectCache<T>, In
168168
) {
169169
this.resourceVersion = '';
170170
} else if (err && (err as { name?: string }).name === 'TimeoutError') {
171-
// Watch client-side timeout — reconnect from last known resourceVersion
171+
// Watch client-side timeout — reconnect from last known resourceVersion.
172+
// The timeout itself already throttled us for the full request timeout, so
173+
// reconnecting immediately cannot tight-loop and backing off would leave
174+
// quiet resources unwatched for up to MAX_RECONNECT_DELAY_MS.
175+
this.reconnectDelayMs = 0;
172176
} else if (err) {
173177
this.callbackCache[ERROR].forEach((elt: ErrorCallback) => elt(err));
174178
return;

‎src/cache_test.ts‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1639,7 +1639,7 @@ describe('ListWatchCache', () => {
16391639
deepStrictEqual(delayValues, [1000]);
16401640
});
16411641

1642-
it('should reconnect with backoff on TimeoutError', async () => {
1642+
it('should reconnect on TimeoutError', async () => {
16431643
const fakeWatch = mock.mock(Watch);
16441644
const listObj = {
16451645
metadata: { resourceVersion: '12345' } as V1ListMeta,
@@ -1682,6 +1682,63 @@ describe('ListWatchCache', () => {
16821682
strictEqual(watchCalls, 2);
16831683
strictEqual(errorEmitted, false);
16841684
});
1685+
1686+
it('should not back off between repeated TimeoutErrors', async () => {
1687+
const fakeWatch = mock.mock(Watch);
1688+
const listObj = {
1689+
metadata: { resourceVersion: '12345' } as V1ListMeta,
1690+
items: [] as V1Namespace[],
1691+
} as V1NamespaceList;
1692+
1693+
const listFn: ListPromise<V1Namespace> = () => Promise.resolve(listObj);
1694+
1695+
let watchCalls = 0;
1696+
const delayValues: number[] = [];
1697+
const promise = new Promise((resolve) => {
1698+
mock.when(
1699+
fakeWatch.watch(mock.anything(), mock.anything(), mock.anything(), mock.anything()),
1700+
).thenCall(() => {
1701+
watchCalls++;
1702+
resolve(new AbortController());
1703+
return Promise.resolve(new AbortController());
1704+
});
1705+
});
1706+
1707+
// ListWatch is constructed for its side effects (starts watching)
1708+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
1709+
const cache = new ListWatch(
1710+
'/some/path',
1711+
mock.instance(fakeWatch),
1712+
listFn,
1713+
true,
1714+
undefined,
1715+
undefined,
1716+
{
1717+
delayFn: (ms: number) => {
1718+
delayValues.push(ms);
1719+
return Promise.resolve();
1720+
},
1721+
},
1722+
);
1723+
await promise;
1724+
strictEqual(watchCalls, 1);
1725+
1726+
const [, , , doneHandler] = mock.capture(fakeWatch.watch).last();
1727+
1728+
const timeoutError = () =>
1729+
new DOMException('The operation was aborted due to timeout', 'TimeoutError');
1730+
1731+
await doneHandler(timeoutError());
1732+
await doneHandler(timeoutError());
1733+
await doneHandler(timeoutError());
1734+
1735+
strictEqual(watchCalls, 4);
1736+
deepStrictEqual(delayValues, []);
1737+
1738+
// Backoff is still applied to non-timeout reconnects.
1739+
await doneHandler(null);
1740+
deepStrictEqual(delayValues, [1000]);
1741+
});
16851742
});
16861743

16871744
describe('delete items', () => {

‎src/watch.ts‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,25 @@ export class Watch {
4040
}
4141

4242
const controller = new AbortController();
43-
const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs);
44-
const signal = AbortSignal.any([controller.signal, timeoutSignal]);
43+
44+
let timedOut: boolean = false;
45+
let timer: NodeJS.Timeout | undefined;
46+
const clearTimer = () => {
47+
if (timer !== undefined) {
48+
clearTimeout(timer);
49+
timer = undefined;
50+
}
51+
};
52+
const resetTimer = () => {
53+
clearTimer();
54+
timer = setTimeout(() => {
55+
timedOut = true;
56+
controller.abort(
57+
new DOMException('The operation was aborted due to timeout', 'TimeoutError'),
58+
);
59+
}, this.requestTimeoutMs);
60+
timer.unref();
61+
};
4562

4663
const ctx = new RequestContext(watchURL.toString(), HttpMethod.GET);
4764
await this.config.applySecurityAuthentication(ctx);
@@ -50,8 +67,9 @@ export class Watch {
5067
const doneCallOnce = (err: any) => {
5168
if (!doneCalled) {
5269
doneCalled = true;
70+
clearTimer();
5371
controller.abort();
54-
if (err && timeoutSignal.aborted) {
72+
if (err && timedOut) {
5573
done(new DOMException('The operation was aborted due to timeout', 'TimeoutError'));
5674
} else {
5775
done(err);
@@ -60,14 +78,17 @@ export class Watch {
6078
};
6179

6280
try {
81+
resetTimer();
6382
const response = await fetch(watchURL, {
6483
method: 'GET',
6584
headers: ctx.getHeaders(),
6685
dispatcher: ctx.getDispatcher(),
67-
signal,
86+
signal: controller.signal,
6887
});
6988

7089
if (response.status === 200) {
90+
// The connect timeout is over; from here on it becomes an inactivity timeout.
91+
resetTimer();
7192
const body = Readable.fromWeb(response.body! as any);
7293

7394
body.on('error', doneCallOnce);
@@ -86,6 +107,7 @@ export class Watch {
86107
// ignore parse errors
87108
}
88109
});
110+
body.on('data', resetTimer);
89111
} else {
90112
const statusText =
91113
response.statusText || STATUS_CODES[response.status] || 'Internal Server Error';

‎src/watch_test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,90 @@ describe('Watch', () => {
386386
strictEqual(doneErr.name, 'TimeoutError');
387387
});
388388

389+
it('should not timeout while the server keeps sending events', async (t) => {
390+
const eventCount = 12;
391+
const eventIntervalMs = 30;
392+
const requestTimeoutMs = 150;
393+
394+
const kc = await setupMockSystem(t, (_req, res) => {
395+
let sent = 0;
396+
const interval = setInterval(() => {
397+
res.write(JSON.stringify({ type: 'ADDED', object: { name: `obj${sent}` } }) + '\n');
398+
sent += 1;
399+
if (sent === eventCount) {
400+
clearInterval(interval);
401+
res.end();
402+
}
403+
}, eventIntervalMs);
404+
});
405+
const watch = new Watch(kc);
406+
407+
// NOTE: Hack around the type system to make the timeout shorter
408+
(watch as any).requestTimeoutMs = requestTimeoutMs;
409+
410+
const receivedObjects: any[] = [];
411+
let doneErr: any;
412+
413+
let doneResolve: () => void;
414+
const donePromise = new Promise<void>((resolve) => {
415+
doneResolve = resolve;
416+
});
417+
418+
await watch.watch(
419+
'/some/path/to/object',
420+
{},
421+
(_phase: string, obj: any) => {
422+
receivedObjects.push(obj);
423+
},
424+
(err: any) => {
425+
doneErr = err;
426+
doneResolve();
427+
},
428+
);
429+
430+
await donePromise;
431+
432+
// The stream lived well past requestTimeoutMs because every event reset the timeout.
433+
strictEqual(receivedObjects.length, eventCount);
434+
strictEqual(doneErr, null);
435+
});
436+
437+
it('should timeout when the server goes silent after connecting', async (t) => {
438+
const kc = await setupMockSystem(t, (_req, res) => {
439+
res.write(JSON.stringify({ type: 'ADDED', object: { name: 'obj' } }) + '\n');
440+
// Then stay silent forever.
441+
});
442+
const watch = new Watch(kc);
443+
444+
// NOTE: Hack around the type system to make the timeout shorter
445+
(watch as any).requestTimeoutMs = 100;
446+
447+
const receivedObjects: any[] = [];
448+
let doneErr: any;
449+
450+
let doneResolve: () => void;
451+
const donePromise = new Promise<void>((resolve) => {
452+
doneResolve = resolve;
453+
});
454+
455+
await watch.watch(
456+
'/some/path/to/object',
457+
{},
458+
(_phase: string, obj: any) => {
459+
receivedObjects.push(obj);
460+
},
461+
(err: any) => {
462+
doneErr = err;
463+
doneResolve();
464+
},
465+
);
466+
467+
await donePromise;
468+
469+
deepStrictEqual(receivedObjects, [{ name: 'obj' }]);
470+
strictEqual(doneErr.name, 'TimeoutError');
471+
});
472+
389473
it('should throw on empty config', async () => {
390474
const kc = new KubeConfig();
391475
const watch = new Watch(kc);

0 commit comments

Comments
 (0)