Skip to content

Commit f2e3c65

Browse files
committed
feat(gax): retry chunk upload on transient errors
Wraps chunk uploads in a retrying executor to retry transient network and server errors using exponential backoff. Retries individual chunks without restarting the entire upload session.
1 parent f2cc9b8 commit f2e3c65

4 files changed

Lines changed: 168 additions & 21 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public ResumableUploadFuture<ResponseT> futureCall(
9191
client.uploadChunkCallable(),
9292
payload,
9393
effectiveSettings,
94-
clientContext.getDefaultCallContext());
94+
clientContext);
9595
}
9696

9797
@Override

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,18 @@
3939
import com.google.api.gax.resumable.ChunkUploadRequest;
4040
import com.google.api.gax.resumable.ChunkUploadResponse;
4141
import com.google.api.gax.resumable.ResumableUploadStatus;
42+
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
43+
import com.google.api.gax.retrying.RetryAlgorithm;
44+
import com.google.api.gax.retrying.RetrySettings;
45+
import com.google.api.gax.retrying.ScheduledRetryingExecutor;
4246
import com.google.common.io.ByteStreams;
4347
import com.google.common.util.concurrent.MoreExecutors;
4448
import java.io.IOException;
4549
import java.io.InputStream;
50+
import java.time.Duration;
4651
import java.util.Arrays;
4752
import java.util.concurrent.CancellationException;
53+
import java.util.concurrent.Executor;
4854
import org.jspecify.annotations.NullMarked;
4955
import org.jspecify.annotations.Nullable;
5056

@@ -57,8 +63,19 @@
5763
@NullMarked
5864
final class ResumableUploadChunkCoordinator<ResponseT> {
5965

66+
private static final RetrySettings CHUNK_RETRY_SETTINGS =
67+
RetrySettings.newBuilder()
68+
.setInitialRetryDelayDuration(Duration.ofMillis(100))
69+
.setRetryDelayMultiplier(1.3)
70+
.setMaxRetryDelayDuration(Duration.ofMinutes(1))
71+
.setMaxAttempts(5)
72+
.build();
73+
6074
private static final byte[] EMPTY_PAYLOAD = new byte[0];
6175

76+
private final Executor chunkExecutor =
77+
MoreExecutors.newSequentialExecutor(MoreExecutors.directExecutor());
78+
6279
private final UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>>
6380
uploadChunkCallable;
6481
private final String uploadUrl;
@@ -74,26 +91,36 @@ final class ResumableUploadChunkCoordinator<ResponseT> {
7491
String uploadUrl,
7592
InputStream payload,
7693
int chunkSize,
77-
ApiCallContext callContext) {
78-
this.uploadChunkCallable =
79-
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
94+
ClientContext clientContext) {
95+
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
8096
this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null");
8197
this.payload = checkNotNull(payload, "payload must not be null");
8298
this.chunkSize = chunkSize;
83-
this.callContext = checkNotNull(callContext, "callContext must not be null");
99+
checkNotNull(clientContext, "clientContext must not be null");
100+
this.callContext = clientContext.getDefaultCallContext();
84101
this.buffer = new byte[chunkSize];
102+
103+
RetryAlgorithm<ChunkUploadResponse<ResponseT>> retryAlgorithm =
104+
new RetryAlgorithm<>(
105+
new ResumableUploadResultRetryAlgorithm<>(ResumableUploadCommand.UPLOAD),
106+
new ExponentialRetryAlgorithm(CHUNK_RETRY_SETTINGS, clientContext.getClock()));
107+
ScheduledRetryingExecutor<ChunkUploadResponse<ResponseT>> retryingExecutor =
108+
new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor());
109+
this.uploadChunkCallable =
110+
new RetryingCallable<>(
111+
clientContext.getDefaultCallContext(), uploadChunkCallable, retryingExecutor);
85112
}
86113

87114
ApiFuture<ResponseT> start() {
88115
result.addListener(
89-
() -> {
90-
ApiFuture<?> chunk = currentChunkFuture;
91-
if (result.isCancelled() && chunk != null) {
92-
chunk.cancel(true);
93-
}
94-
},
95-
MoreExecutors.directExecutor());
96-
transmitChunk(0L);
116+
() -> {
117+
ApiFuture<?> chunk = currentChunkFuture;
118+
if (result.isCancelled() && chunk != null) {
119+
chunk.cancel(true);
120+
}
121+
},
122+
MoreExecutors.directExecutor());
123+
chunkExecutor.execute(() -> transmitChunk(0L));
97124
return result;
98125
}
99126

@@ -153,9 +180,10 @@ public void onSuccess(ChunkUploadResponse<ResponseT> response) {
153180
result.setException(
154181
new IllegalStateException(
155182
"Upload stream ended and final chunk was transmitted, but server returned"
156-
+ " incomplete status"));
183+
+ " incomplete status for upload URL: "
184+
+ uploadUrl));
157185
} else {
158-
transmitChunk(nextOffset);
186+
chunkExecutor.execute(() -> transmitChunk(nextOffset));
159187
}
160188
}
161189

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ final class ResumableUploadFutureImpl<ResponseT> implements ResumableUploadFutur
6767
uploadChunkCallable;
6868
private final InputStream payload;
6969
private final ResumableUploadCallSettings settings;
70-
private final ApiCallContext callContext;
70+
private final ClientContext clientContext;
7171
private final SettableApiFuture<ResponseT> resultFuture = SettableApiFuture.create();
7272

7373
private volatile @Nullable String uploadSessionUrl;
@@ -87,10 +87,10 @@ static <ResponseT> ResumableUploadFutureImpl<ResponseT> create(
8787
UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
8888
InputStream payload,
8989
ResumableUploadCallSettings settings,
90-
ApiCallContext callContext) {
90+
ClientContext clientContext) {
9191
ResumableUploadFutureImpl<ResponseT> future =
9292
new ResumableUploadFutureImpl<>(
93-
startFuture, uploadChunkCallable, payload, settings, callContext);
93+
startFuture, uploadChunkCallable, payload, settings, clientContext);
9494
try {
9595
future.start();
9696
} catch (Throwable t) {
@@ -104,14 +104,14 @@ private ResumableUploadFutureImpl(
104104
UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
105105
InputStream payload,
106106
ResumableUploadCallSettings settings,
107-
ApiCallContext callContext) {
107+
ClientContext clientContext) {
108108
this.startFuture = checkNotNull(startFuture, "startFuture must not be null");
109109
this.uploadChunkCallable =
110110
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
111111
this.payload = checkNotNull(payload, "payload must not be null");
112112
this.settings = checkNotNull(settings, "settings must not be null");
113113
checkArgument(settings.getChunkSize() > 0, "chunkSize must be > 0");
114-
this.callContext = checkNotNull(callContext, "callContext must not be null");
114+
this.clientContext = checkNotNull(clientContext, "clientContext must not be null");
115115
this.inFlightFuture = startFuture;
116116
}
117117

@@ -131,7 +131,7 @@ public void onSuccess(ResumableUploadSession session) {
131131
uploadSessionUrl,
132132
payload,
133133
settings.getChunkSize(),
134-
callContext);
134+
clientContext);
135135
ApiFuture<ResponseT> uploadFuture;
136136
try {
137137
uploadFuture = coordinator.start();

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,115 @@ void testResumeCall_throwsUnsupportedOperationException() {
404404
() -> callable.resumeCall("https://upload.url/session", streamOf("data"), null));
405405
}
406406

407+
@Test
408+
void testChunkRetry_transientFailureThenSuccess_retriesAndSucceeds() throws Exception {
409+
stubStartSession("https://upload.url/chunk-retry-ok");
410+
TrackableStream stream = new TrackableStream("01234567"); // exactly 1 chunk of 8 bytes
411+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
412+
.thenReturn(
413+
ApiFutures.immediateFailedFuture(createApiException(503, StatusCode.Code.UNAVAILABLE)))
414+
.thenReturn(
415+
ApiFutures.immediateFuture(
416+
ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "chunk-done")));
417+
418+
ResumableUploadFuture<String> future = callable.futureCall("resource-path", stream, null);
419+
420+
assertThat(future.get()).isEqualTo("chunk-done");
421+
assertThat(future.isDone()).isTrue();
422+
assertThat(stream.totalBytesRead).isEqualTo(8);
423+
assertThat(stream.closed).isTrue();
424+
425+
ArgumentCaptor<ChunkUploadRequest> captor = ArgumentCaptor.forClass(ChunkUploadRequest.class);
426+
verify(mockChunkCallable, times(2)).futureCall(captor.capture(), any());
427+
List<ChunkUploadRequest> requests = captor.getAllValues();
428+
assertThat(requests.get(0).getOffset()).isEqualTo(0);
429+
assertThat(requests.get(0).getPayload()).isEqualTo("01234567".getBytes(StandardCharsets.UTF_8));
430+
assertThat(requests.get(1).getOffset()).isEqualTo(0);
431+
assertThat(requests.get(1).getPayload()).isEqualTo("01234567".getBytes(StandardCharsets.UTF_8));
432+
}
433+
434+
@Test
435+
void testChunkRetry_transientFailureExhaustion_surfacesLastError() {
436+
stubStartSession("https://upload.url/chunk-exhaustion");
437+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
438+
.thenReturn(
439+
ApiFutures.immediateFailedFuture(createApiException(503, StatusCode.Code.UNAVAILABLE)));
440+
441+
ResumableUploadFuture<String> future =
442+
callable.futureCall("resource-path", streamOf("hello"), null);
443+
444+
ExecutionException exception = assertThrows(ExecutionException.class, future::get);
445+
assertThat(exception.getCause()).isInstanceOf(ApiException.class);
446+
assertThat(((ApiException) exception.getCause()).getStatusCode().getTransportCode())
447+
.isEqualTo(503);
448+
449+
// Default chunk retry settings has maxAttempts = 5
450+
verify(mockChunkCallable, times(5)).futureCall(any(), any());
451+
}
452+
453+
@Test
454+
void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() {
455+
SettableApiFuture<ChunkUploadResponse<String>> chunkAttempt0Future = SettableApiFuture.create();
456+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
457+
.thenReturn(chunkAttempt0Future)
458+
.thenReturn(
459+
ApiFutures.immediateFuture(
460+
ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "should-not-reach")));
461+
462+
ClientContext clientContext =
463+
ClientContext.newBuilder().setDefaultCallContext(callContext).build();
464+
SettableApiFuture<ResumableUploadSession> startSessionFuture = SettableApiFuture.create();
465+
startSessionFuture.set(
466+
ResumableUploadSession.newBuilder()
467+
.setUploadUrl("https://upload.url/cancel-backoff")
468+
.build());
469+
ResumableUploadCallSettings customSettings =
470+
defaultSettings.toBuilder().setChunkSize(8).build();
471+
ResumableUploadFutureImpl<String> sessionFuture =
472+
ResumableUploadFutureImpl.create(
473+
startSessionFuture,
474+
mockChunkCallable,
475+
streamOf("hello"),
476+
customSettings,
477+
clientContext);
478+
479+
// Fail attempt 0 with 503 to schedule backoff
480+
chunkAttempt0Future.setException(createApiException(503, StatusCode.Code.UNAVAILABLE));
481+
482+
// Cancel while backoff is pending
483+
assertThat(sessionFuture.cancel(true)).isTrue();
484+
assertThat(sessionFuture.isCancelled()).isTrue();
485+
assertThrows(CancellationException.class, sessionFuture::get);
486+
487+
// Only attempt 0 occurred; attempt 1 was de-scheduled
488+
verify(mockChunkCallable, times(1)).futureCall(any(), any());
489+
}
490+
491+
private static class HttpStatusStatusCode implements StatusCode {
492+
private final int httpStatus;
493+
private final StatusCode.Code code;
494+
495+
HttpStatusStatusCode(int httpStatus, StatusCode.Code code) {
496+
this.httpStatus = httpStatus;
497+
this.code = code;
498+
}
499+
500+
@Override
501+
public StatusCode.Code getCode() {
502+
return code;
503+
}
504+
505+
@Override
506+
public Integer getTransportCode() {
507+
return httpStatus;
508+
}
509+
}
510+
511+
private static ApiException createApiException(int httpStatus, StatusCode.Code code) {
512+
return ApiExceptionFactory.createException(
513+
"HTTP " + httpStatus, null, new HttpStatusStatusCode(httpStatus, code), false);
514+
}
515+
407516
private void stubStartSession(String uploadUrl) {
408517
when(mockStartCallable.futureCall(any(), any()))
409518
.thenReturn(
@@ -424,11 +533,21 @@ private static void assertChunk(
424533

425534
private static class TrackableStream extends ByteArrayInputStream {
426535
boolean closed = false;
536+
int totalBytesRead = 0;
427537

428538
TrackableStream(String content) {
429539
super(content.getBytes(StandardCharsets.UTF_8));
430540
}
431541

542+
@Override
543+
public int read(byte[] b, int off, int len) {
544+
int read = super.read(b, off, len);
545+
if (read > 0) {
546+
totalBytesRead += read;
547+
}
548+
return read;
549+
}
550+
432551
@Override
433552
public void close() throws IOException {
434553
closed = true;

0 commit comments

Comments
 (0)