Skip to content

Commit c4f3781

Browse files
Merge pull request #3034 from brendandburns/fix-informer-retryable-errors
Retry transient informer failures
2 parents 93f0cb9 + 04cee40 commit c4f3781

2 files changed

Lines changed: 147 additions & 17 deletions

File tree

‎src/cache.ts‎

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -197,8 +197,31 @@ export class ListWatch<T extends KubernetesObject> implements ObjectCache<T>, In
197197
return ms + this.randFn() * ListWatch.BACKOFF_JITTER * ms;
198198
}
199199

200+
private errorStatusCode(err: any): number | undefined {
201+
const statusCode = err?.statusCode ?? err?.code;
202+
return typeof statusCode === 'number' ? statusCode : undefined;
203+
}
204+
205+
private isRetryableError(err: any): boolean {
206+
const statusCode = this.errorStatusCode(err);
207+
return statusCode === undefined || statusCode === 408 || statusCode === 429 || statusCode >= 500;
208+
}
209+
210+
private async waitForRetry(): Promise<boolean> {
211+
if (this.reconnectDelayMs === 0) {
212+
this.reconnectDelayMs = this.nextBackoffLevelMs();
213+
}
214+
await this.delayFn(this.withJitter(this.reconnectDelayMs));
215+
if (this.stopped) {
216+
return false;
217+
}
218+
this.reconnectDelayMs = this.nextBackoffLevelMs();
219+
return true;
220+
}
221+
200222
private async doneHandler(err: any): Promise<void> {
201223
this._stop();
224+
let hasBackedOff = false;
202225
if (
203226
err &&
204227
((err as { statusCode?: number }).statusCode === 410 || (err as { code?: number }).code === 410)
@@ -212,25 +235,36 @@ export class ListWatch<T extends KubernetesObject> implements ObjectCache<T>, In
212235
this.reconnectDelayMs = 0;
213236
} else if (err) {
214237
this.callbackCache[ERROR].forEach((elt: ErrorCallback) => elt(err));
215-
return;
238+
if (!this.isRetryableError(err) || this.stopped || !(await this.waitForRetry())) {
239+
return;
240+
}
241+
hasBackedOff = true;
216242
}
217243
if (this.stopped) {
218244
// do not auto-restart
219245
return;
220246
}
221247
this.callbackCache[CONNECT].forEach((elt: ErrorCallback) => elt(undefined));
222248
if (!this.resourceVersion) {
223-
let list: KubernetesListObject<T>;
224-
try {
225-
const promise = this.listFn();
226-
list = await promise;
227-
} catch (err) {
228-
this.callbackCache[ERROR].forEach((elt: ErrorCallback) => elt(err));
229-
return;
249+
let listed = false;
250+
while (!listed) {
251+
let list: KubernetesListObject<T>;
252+
try {
253+
const promise = this.listFn();
254+
list = await promise;
255+
} catch (err) {
256+
this.callbackCache[ERROR].forEach((elt: ErrorCallback) => elt(err));
257+
if (!this.isRetryableError(err) || this.stopped || !(await this.waitForRetry())) {
258+
return;
259+
}
260+
hasBackedOff = true;
261+
continue;
262+
}
263+
this.objects = deleteItems(this.objects, list.items, this.callbackCache[DELETE].slice());
264+
this.addOrUpdateItems(list.items);
265+
this.resourceVersion = list.metadata ? list.metadata!.resourceVersion || '' : '';
266+
listed = true;
230267
}
231-
this.objects = deleteItems(this.objects, list.items, this.callbackCache[DELETE].slice());
232-
this.addOrUpdateItems(list.items);
233-
this.resourceVersion = list.metadata ? list.metadata!.resourceVersion || '' : '';
234268
}
235269
const queryParams = {
236270
resourceVersion: this.resourceVersion,
@@ -245,10 +279,13 @@ export class ListWatch<T extends KubernetesObject> implements ObjectCache<T>, In
245279
if (this.fieldSelector !== undefined) {
246280
queryParams.fieldSelector = ObjectSerializer.serialize(this.fieldSelector, 'string');
247281
}
248-
if (this.hasConnected) {
282+
if (this.hasConnected && !hasBackedOff) {
249283
if (this.reconnectDelayMs > 0) {
250284
await this.delayFn(this.withJitter(this.reconnectDelayMs));
251285
}
286+
if (this.stopped) {
287+
return;
288+
}
252289
this.reconnectDelayMs = this.nextBackoffLevelMs();
253290
}
254291
this.hasConnected = true;

‎src/cache_test.ts‎

Lines changed: 98 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1106,7 +1106,7 @@ describe('ListWatchCache', () => {
11061106
).twice();
11071107
});
11081108

1109-
it('does not auto-restart after an error', async () => {
1109+
it('does not auto-restart after a non-retryable error', async () => {
11101110
const fakeWatch = mock.mock(Watch);
11111111
const list: V1Pod[] = [
11121112
{
@@ -1150,7 +1150,7 @@ describe('ListWatchCache', () => {
11501150

11511151
const [, , , doneHandler] = mock.capture(fakeWatch.watch).last();
11521152

1153-
const error = new Error('testing');
1153+
const error = Object.assign(new Error('Bad Request'), { statusCode: 400 });
11541154
await doneHandler(error);
11551155

11561156
mock.verify(
@@ -1523,6 +1523,7 @@ describe('ListWatchCache', () => {
15231523
},
15241524
});
15251525

1526+
await informer.stop();
15261527
mockAgent.assertNoPendingInterceptors();
15271528
});
15281529

@@ -1686,6 +1687,98 @@ describe('ListWatchCache', () => {
16861687
strictEqual(errorEmitted, false);
16871688
});
16881689

1690+
it('should reconnect with backoff after a retryable watch error', async () => {
1691+
const fakeWatch = mock.mock(Watch);
1692+
const listObj = {
1693+
metadata: { resourceVersion: '12345' } as V1ListMeta,
1694+
items: [] as V1Namespace[],
1695+
} as V1NamespaceList;
1696+
const listFn: ListPromise<V1Namespace> = () => Promise.resolve(listObj);
1697+
1698+
let watchCalls = 0;
1699+
const delays: number[] = [];
1700+
mock.when(
1701+
fakeWatch.watch(mock.anything(), mock.anything(), mock.anything(), mock.anything()),
1702+
).thenCall(() => {
1703+
watchCalls++;
1704+
return Promise.resolve(new AbortController());
1705+
});
1706+
1707+
const informer = new ListWatch(
1708+
'/some/path',
1709+
mock.instance(fakeWatch),
1710+
listFn,
1711+
false,
1712+
undefined,
1713+
undefined,
1714+
{
1715+
delayFn: (ms: number) => {
1716+
delays.push(ms);
1717+
return Promise.resolve();
1718+
},
1719+
randFn: () => 0,
1720+
},
1721+
);
1722+
const errors: Error[] = [];
1723+
informer.on('error', (err) => errors.push(err));
1724+
await informer.start();
1725+
1726+
const [, , , doneHandler] = mock.capture(fakeWatch.watch).last();
1727+
const error = Object.assign(new Error('Service Unavailable'), { statusCode: 503 });
1728+
await doneHandler(error);
1729+
1730+
strictEqual(watchCalls, 2);
1731+
deepStrictEqual(delays, [800]);
1732+
deepStrictEqual(errors, [error]);
1733+
});
1734+
1735+
it('should retry the initial list with backoff after a retryable error', async () => {
1736+
const fakeWatch = mock.mock(Watch);
1737+
const listObj = {
1738+
metadata: { resourceVersion: '12345' } as V1ListMeta,
1739+
items: [] as V1Namespace[],
1740+
} as V1NamespaceList;
1741+
const error = Object.assign(new Error('Too Many Requests'), { code: 429 });
1742+
let listCalls = 0;
1743+
const listFn: ListPromise<V1Namespace> = () => {
1744+
listCalls++;
1745+
return listCalls === 1 ? Promise.reject(error) : Promise.resolve(listObj);
1746+
};
1747+
const delays: number[] = [];
1748+
let watchCalls = 0;
1749+
mock.when(
1750+
fakeWatch.watch(mock.anything(), mock.anything(), mock.anything(), mock.anything()),
1751+
).thenCall(() => {
1752+
watchCalls++;
1753+
return Promise.resolve(new AbortController());
1754+
});
1755+
1756+
const informer = new ListWatch(
1757+
'/some/path',
1758+
mock.instance(fakeWatch),
1759+
listFn,
1760+
false,
1761+
undefined,
1762+
undefined,
1763+
{
1764+
delayFn: (ms: number) => {
1765+
delays.push(ms);
1766+
return Promise.resolve();
1767+
},
1768+
randFn: () => 0,
1769+
},
1770+
);
1771+
const errors: Error[] = [];
1772+
informer.on('error', (err) => errors.push(err));
1773+
1774+
await informer.start();
1775+
1776+
strictEqual(listCalls, 2);
1777+
strictEqual(watchCalls, 1);
1778+
deepStrictEqual(delays, [800]);
1779+
deepStrictEqual(errors, [error]);
1780+
});
1781+
16891782
it('should not back off between repeated TimeoutErrors', async () => {
16901783
const fakeWatch = mock.mock(Watch);
16911784
const listObj = {
@@ -1896,7 +1989,7 @@ describe('delete items', () => {
18961989
strictEqual(await connectPromise, true);
18971990
});
18981991

1899-
it('does calls connect after a restart after an error', async () => {
1992+
it('calls connect after manually restarting from a non-retryable error', async () => {
19001993
const fakeWatch = mock.mock(Watch);
19011994
const list: V1Pod[] = [
19021995
{
@@ -1940,7 +2033,7 @@ describe('delete items', () => {
19402033

19412034
const [, , , doneHandler] = mock.capture(fakeWatch.watch).last();
19422035

1943-
const error = new Error('testing');
2036+
const error = Object.assign(new Error('Bad Request'), { statusCode: 400 });
19442037
await doneHandler(error);
19452038

19462039
mock.verify(
@@ -1960,7 +2053,7 @@ describe('delete items', () => {
19602053

19612054
it('should correctly handle errors in the initial list', async () => {
19622055
const fake = mock.mock(Watch);
1963-
const requestErr = Error('request failed');
2056+
const requestErr = Object.assign(Error('request failed'), { statusCode: 400 });
19642057
const listFn: ListPromise<V1Namespace> = function (): Promise<V1NamespaceList> {
19652058
return new Promise<V1NamespaceList>((resolve, reject) => {
19662059
reject(requestErr);

0 commit comments

Comments
 (0)