Skip to content

Commit b5938fa

Browse files
Fix informer recovery from ResourceVersionTooLarge (closes #2190)
1 parent 50ba90f commit b5938fa

2 files changed

Lines changed: 132 additions & 1 deletion

File tree

‎util/src/main/java/io/kubernetes/client/informer/cache/ReflectorRunnable.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@
1919
import io.kubernetes.client.informer.ListerWatcher;
2020
import io.kubernetes.client.informer.exception.WatchExpiredException;
2121
import io.kubernetes.client.openapi.ApiException;
22+
import io.kubernetes.client.openapi.JSON;
2223
import io.kubernetes.client.openapi.models.V1ListMeta;
2324
import io.kubernetes.client.openapi.models.V1ObjectMeta;
25+
import io.kubernetes.client.openapi.models.V1Status;
2426
import io.kubernetes.client.util.CallGeneratorParams;
2527
import io.kubernetes.client.util.ModelMapper;
2628
import io.kubernetes.client.util.Strings;
@@ -182,7 +184,7 @@ public void run() {
182184
}
183185
}
184186
} catch (ApiException e) {
185-
if (e.getCode() == HttpURLConnection.HTTP_GONE) {
187+
if (e.getCode() == HttpURLConnection.HTTP_GONE || isTooLargeResourceVersion(e)) {
186188
log.info(
187189
"ResourceVersion {} expired, will retry w/o resourceVersion at the next time",
188190
getRelistResourceVersion());
@@ -195,6 +197,24 @@ public void run() {
195197
}
196198
}
197199

200+
private static boolean isTooLargeResourceVersion(ApiException exception) {
201+
if (Strings.isNullOrEmpty(exception.getResponseBody())) {
202+
return false;
203+
}
204+
V1Status status;
205+
try {
206+
status = JSON.deserialize(exception.getResponseBody(), V1Status.class);
207+
} catch (RuntimeException e) {
208+
// Generated status adapters can throw unchecked exceptions for malformed response bodies.
209+
return false;
210+
}
211+
return status != null
212+
&& status.getDetails() != null
213+
&& status.getDetails().getCauses() != null
214+
&& status.getDetails().getCauses().stream()
215+
.anyMatch(cause -> cause != null && "ResourceVersionTooLarge".equals(cause.getReason()));
216+
}
217+
198218
public void stop() {
199219
try {
200220
isActive.set(false);

‎util/src/test/java/io/kubernetes/client/informer/cache/ReflectorRunnableTest.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@
4343
import org.awaitility.Awaitility;
4444
import org.junit.jupiter.api.Test;
4545
import org.junit.jupiter.api.extension.ExtendWith;
46+
import org.junit.jupiter.params.ParameterizedTest;
47+
import org.junit.jupiter.params.provider.NullAndEmptySource;
48+
import org.junit.jupiter.params.provider.ValueSource;
49+
import org.mockito.ArgumentCaptor;
4650
import org.mockito.Mock;
4751
import org.mockito.junit.jupiter.MockitoExtension;
4852

@@ -212,6 +216,113 @@ void reflectorRunnableShouldCaptureListNon410ApiException() throws ApiException
212216
}
213217
}
214218

219+
@ParameterizedTest
220+
@ValueSource(ints = {410, 504})
221+
void reflectorRecoversUnavailableResourceVersion(int statusCode) throws ApiException {
222+
V1Pod initialPod = new V1Pod().metadata(new V1ObjectMeta().name("initial"));
223+
V1Pod recoveredPod = new V1Pod().metadata(new V1ObjectMeta().name("recovered"));
224+
ApiException unavailable =
225+
statusCode == 410
226+
? new ApiException(410, "Gone")
227+
: new ApiException(
228+
statusCode,
229+
null,
230+
"{\"kind\":\"Status\",\"apiVersion\":\"v1\",\"reason\":\"Timeout\","
231+
+ "\"details\":{\"causes\":[{\"reason\":\"Other\"},"
232+
+ "{\"reason\":\"ResourceVersionTooLarge\"}]},\"code\":504}");
233+
when(listerWatcher.list(any()))
234+
.thenReturn(
235+
new V1PodList()
236+
.metadata(new V1ListMeta().resourceVersion("100"))
237+
.items(Arrays.asList(initialPod)))
238+
.thenThrow(unavailable)
239+
.thenReturn(
240+
new V1PodList()
241+
.metadata(new V1ListMeta().resourceVersion("200"))
242+
.items(Arrays.asList(recoveredPod)));
243+
MockWatch<V1Pod> initialWatch =
244+
new MockWatch<>(new Watch.Response<>(EventType.ERROR.name(), new V1Status().code(500)));
245+
MockWatch<V1Pod> recoveredWatch =
246+
new MockWatch<>(
247+
new Watch.Response<>(
248+
EventType.BOOKMARK.name(),
249+
new V1Pod().metadata(new V1ObjectMeta().resourceVersion("201"))),
250+
new Watch.Response<>(EventType.ERROR.name(), new V1Status().code(500)));
251+
when(listerWatcher.watch(any())).thenReturn(initialWatch, recoveredWatch);
252+
ReflectorRunnable<V1Pod, V1PodList> reflector =
253+
new ReflectorRunnable<>(V1Pod.class, listerWatcher, deltaFIFO, exceptionHandler);
254+
255+
reflector.run();
256+
assertThat(reflector.getLastSyncResourceVersion()).isEqualTo("100");
257+
reflector.run();
258+
assertThat(reflector.isLastSyncResourceVersionUnavailable()).isTrue();
259+
assertThat(reflector.getLastSyncResourceVersion()).isEqualTo("100");
260+
verify(exceptionHandler, never()).accept(V1Pod.class, unavailable);
261+
reflector.run();
262+
263+
ArgumentCaptor<CallGeneratorParams> listParams =
264+
ArgumentCaptor.forClass(CallGeneratorParams.class);
265+
verify(listerWatcher, times(3)).list(listParams.capture());
266+
assertThat(listParams.getAllValues())
267+
.extracting(params -> params.resourceVersion)
268+
.containsExactly("0", "100", "");
269+
ArgumentCaptor<CallGeneratorParams> watchParams =
270+
ArgumentCaptor.forClass(CallGeneratorParams.class);
271+
verify(listerWatcher, times(2)).watch(watchParams.capture());
272+
assertThat(watchParams.getAllValues())
273+
.extracting(params -> params.resourceVersion)
274+
.containsExactly("100", "200");
275+
verify(deltaFIFO).replace(Arrays.asList(initialPod), "100");
276+
verify(deltaFIFO).replace(Arrays.asList(recoveredPod), "200");
277+
assertThat(reflector.isLastSyncResourceVersionUnavailable()).isFalse();
278+
assertThat(reflector.getLastSyncResourceVersion()).isEqualTo("201");
279+
assertThat(initialWatch.isClosed()).isTrue();
280+
assertThat(recoveredWatch.isClosed()).isTrue();
281+
}
282+
283+
@ParameterizedTest
284+
@NullAndEmptySource
285+
@ValueSource(
286+
strings = {
287+
"not json",
288+
"null",
289+
"[]",
290+
"{}",
291+
"{\"details\":null}",
292+
"{\"details\":{\"causes\":null}}",
293+
"{\"details\":{\"causes\":[]}}",
294+
"{\"details\":{\"causes\":[null,{}]}}",
295+
"{\"details\":{\"causes\":\"invalid\"}}",
296+
"{\"details\":{\"causes\":[{\"reason\":{}}]}}",
297+
"{\"reason\":\"Timeout\",\"message\":\"Too large resource version\"}",
298+
"{\"reason\":\"Timeout\",\"details\":{\"causes\":[{\"reason\":\"Other\"}]}}"
299+
})
300+
void reflectorReportsOrdinaryTimeoutsAndMalformedBodies(String body) throws ApiException {
301+
ApiException timeout = new ApiException(504, null, body);
302+
when(listerWatcher.list(any()))
303+
.thenReturn(new V1PodList().metadata(new V1ListMeta().resourceVersion("100")))
304+
.thenThrow(timeout);
305+
when(listerWatcher.watch(any()))
306+
.thenReturn(
307+
new MockWatch<>(
308+
new Watch.Response<>(EventType.ERROR.name(), new V1Status().code(500))));
309+
ReflectorRunnable<V1Pod, V1PodList> reflector =
310+
new ReflectorRunnable<>(V1Pod.class, listerWatcher, deltaFIFO, exceptionHandler);
311+
312+
reflector.run();
313+
reflector.run();
314+
reflector.run();
315+
316+
ArgumentCaptor<CallGeneratorParams> params = ArgumentCaptor.forClass(CallGeneratorParams.class);
317+
verify(listerWatcher, times(3)).list(params.capture());
318+
assertThat(params.getAllValues())
319+
.extracting(value -> value.resourceVersion)
320+
.containsExactly("0", "100", "100");
321+
verify(exceptionHandler, times(2)).accept(V1Pod.class, timeout);
322+
assertThat(reflector.isLastSyncResourceVersionUnavailable()).isFalse();
323+
assertThat(reflector.getLastSyncResourceVersion()).isEqualTo("100");
324+
}
325+
215326
@Test
216327
void reflectorRunnableCaptureWatchException() throws ApiException {
217328
RuntimeException expectedException = new RuntimeException("noxu");

0 commit comments

Comments
 (0)