Skip to content

Commit 79b57f6

Browse files
committed
feat(gax): retry chunk upload on transient errors
Wraps chunk uploads in a retrying executor to retry transient network and server errors using exponential backoff. Retries individual chunks without restarting the entire upload session.
1 parent f2cc9b8 commit 79b57f6

4 files changed

Lines changed: 173 additions & 20 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public ResumableUploadFuture<ResponseT> futureCall(
9191
client.uploadChunkCallable(),
9292
payload,
9393
effectiveSettings,
94-
clientContext.getDefaultCallContext());
94+
clientContext);
9595
}
9696

9797
@Override

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,18 @@
3939
import com.google.api.gax.resumable.ChunkUploadRequest;
4040
import com.google.api.gax.resumable.ChunkUploadResponse;
4141
import com.google.api.gax.resumable.ResumableUploadStatus;
42+
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
43+
import com.google.api.gax.retrying.RetryAlgorithm;
44+
import com.google.api.gax.retrying.RetrySettings;
45+
import com.google.api.gax.retrying.ScheduledRetryingExecutor;
4246
import com.google.common.io.ByteStreams;
4347
import com.google.common.util.concurrent.MoreExecutors;
4448
import java.io.IOException;
4549
import java.io.InputStream;
50+
import java.time.Duration;
4651
import java.util.Arrays;
4752
import java.util.concurrent.CancellationException;
53+
import java.util.concurrent.Executor;
4854
import org.jspecify.annotations.NullMarked;
4955
import org.jspecify.annotations.Nullable;
5056

@@ -57,8 +63,23 @@
5763
@NullMarked
5864
final class ResumableUploadChunkCoordinator<ResponseT> {
5965

66+
private static final RetrySettings CHUNK_RETRY_SETTINGS =
67+
RetrySettings.newBuilder()
68+
.setInitialRetryDelayDuration(Duration.ofMillis(100))
69+
.setRetryDelayMultiplier(1.3)
70+
.setMaxRetryDelayDuration(Duration.ofMinutes(1))
71+
.setInitialRpcTimeoutDuration(Duration.ofSeconds(30))
72+
.setRpcTimeoutMultiplier(1.0)
73+
.setMaxRpcTimeoutDuration(Duration.ofSeconds(30))
74+
.setTotalTimeoutDuration(Duration.ofMinutes(5))
75+
.setMaxAttempts(5)
76+
.build();
77+
6078
private static final byte[] EMPTY_PAYLOAD = new byte[0];
6179

80+
private final Executor chunkExecutor =
81+
MoreExecutors.newSequentialExecutor(MoreExecutors.directExecutor());
82+
6283
private final UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>>
6384
uploadChunkCallable;
6485
private final String uploadUrl;
@@ -74,26 +95,36 @@ final class ResumableUploadChunkCoordinator<ResponseT> {
7495
String uploadUrl,
7596
InputStream payload,
7697
int chunkSize,
77-
ApiCallContext callContext) {
78-
this.uploadChunkCallable =
79-
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
98+
ClientContext clientContext) {
99+
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
80100
this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null");
81101
this.payload = checkNotNull(payload, "payload must not be null");
82102
this.chunkSize = chunkSize;
83-
this.callContext = checkNotNull(callContext, "callContext must not be null");
103+
checkNotNull(clientContext, "clientContext must not be null");
104+
this.callContext = clientContext.getDefaultCallContext();
84105
this.buffer = new byte[chunkSize];
106+
107+
RetryAlgorithm<ChunkUploadResponse<ResponseT>> retryAlgorithm =
108+
new RetryAlgorithm<>(
109+
new ResumableUploadResultRetryAlgorithm<>(ResumableUploadCommand.UPLOAD),
110+
new ExponentialRetryAlgorithm(CHUNK_RETRY_SETTINGS, clientContext.getClock()));
111+
ScheduledRetryingExecutor<ChunkUploadResponse<ResponseT>> retryingExecutor =
112+
new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor());
113+
this.uploadChunkCallable =
114+
new RetryingCallable<>(
115+
clientContext.getDefaultCallContext(), uploadChunkCallable, retryingExecutor);
85116
}
86117

87118
ApiFuture<ResponseT> start() {
88119
result.addListener(
89-
() -> {
90-
ApiFuture<?> chunk = currentChunkFuture;
91-
if (result.isCancelled() && chunk != null) {
92-
chunk.cancel(true);
93-
}
94-
},
95-
MoreExecutors.directExecutor());
96-
transmitChunk(0L);
120+
() -> {
121+
ApiFuture<?> chunk = currentChunkFuture;
122+
if (result.isCancelled() && chunk != null) {
123+
chunk.cancel(true);
124+
}
125+
},
126+
MoreExecutors.directExecutor());
127+
chunkExecutor.execute(() -> transmitChunk(0L));
97128
return result;
98129
}
99130

@@ -153,9 +184,10 @@ public void onSuccess(ChunkUploadResponse<ResponseT> response) {
153184
result.setException(
154185
new IllegalStateException(
155186
"Upload stream ended and final chunk was transmitted, but server returned"
156-
+ " incomplete status"));
187+
+ " incomplete status for upload URL: "
188+
+ uploadUrl));
157189
} else {
158-
transmitChunk(nextOffset);
190+
chunkExecutor.execute(() -> transmitChunk(nextOffset));
159191
}
160192
}
161193

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ final class ResumableUploadFutureImpl<ResponseT> implements ResumableUploadFutur
6868
private final InputStream payload;
6969
private final ResumableUploadCallSettings settings;
7070
private final ApiCallContext callContext;
71+
private final ClientContext clientContext;
7172
private final SettableApiFuture<ResponseT> resultFuture = SettableApiFuture.create();
7273

7374
private volatile @Nullable String uploadSessionUrl;
@@ -87,10 +88,10 @@ static <ResponseT> ResumableUploadFutureImpl<ResponseT> create(
8788
UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
8889
InputStream payload,
8990
ResumableUploadCallSettings settings,
90-
ApiCallContext callContext) {
91+
ClientContext clientContext) {
9192
ResumableUploadFutureImpl<ResponseT> future =
9293
new ResumableUploadFutureImpl<>(
93-
startFuture, uploadChunkCallable, payload, settings, callContext);
94+
startFuture, uploadChunkCallable, payload, settings, clientContext);
9495
try {
9596
future.start();
9697
} catch (Throwable t) {
@@ -104,14 +105,15 @@ private ResumableUploadFutureImpl(
104105
UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
105106
InputStream payload,
106107
ResumableUploadCallSettings settings,
107-
ApiCallContext callContext) {
108+
ClientContext clientContext) {
108109
this.startFuture = checkNotNull(startFuture, "startFuture must not be null");
109110
this.uploadChunkCallable =
110111
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
111112
this.payload = checkNotNull(payload, "payload must not be null");
112113
this.settings = checkNotNull(settings, "settings must not be null");
113114
checkArgument(settings.getChunkSize() > 0, "chunkSize must be > 0");
114-
this.callContext = checkNotNull(callContext, "callContext must not be null");
115+
this.clientContext = checkNotNull(clientContext, "clientContext must not be null");
116+
this.callContext = clientContext.getDefaultCallContext();
115117
this.inFlightFuture = startFuture;
116118
}
117119

@@ -131,7 +133,7 @@ public void onSuccess(ResumableUploadSession session) {
131133
uploadSessionUrl,
132134
payload,
133135
settings.getChunkSize(),
134-
callContext);
136+
clientContext);
135137
ApiFuture<ResponseT> uploadFuture;
136138
try {
137139
uploadFuture = coordinator.start();

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ResumableUploadCallableImplTest.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,115 @@ void testResumeCall_throwsUnsupportedOperationException() {
404404
() -> callable.resumeCall("https://upload.url/session", streamOf("data"), null));
405405
}
406406

407+
@Test
408+
void testChunkRetry_transientFailureThenSuccess_retriesAndSucceeds() throws Exception {
409+
stubStartSession("https://upload.url/chunk-retry-ok");
410+
TrackableStream stream = new TrackableStream("01234567"); // exactly 1 chunk of 8 bytes
411+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
412+
.thenReturn(
413+
ApiFutures.immediateFailedFuture(createApiException(503, StatusCode.Code.UNAVAILABLE)))
414+
.thenReturn(
415+
ApiFutures.immediateFuture(
416+
ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "chunk-done")));
417+
418+
ResumableUploadFuture<String> future = callable.futureCall("resource-path", stream, null);
419+
420+
assertThat(future.get()).isEqualTo("chunk-done");
421+
assertThat(future.isDone()).isTrue();
422+
assertThat(stream.totalBytesRead).isEqualTo(8);
423+
assertThat(stream.closed).isTrue();
424+
425+
ArgumentCaptor<ChunkUploadRequest> captor = ArgumentCaptor.forClass(ChunkUploadRequest.class);
426+
verify(mockChunkCallable, times(2)).futureCall(captor.capture(), any());
427+
List<ChunkUploadRequest> requests = captor.getAllValues();
428+
assertThat(requests.get(0).getOffset()).isEqualTo(0);
429+
assertThat(requests.get(0).getPayload()).isEqualTo("01234567".getBytes(StandardCharsets.UTF_8));
430+
assertThat(requests.get(1).getOffset()).isEqualTo(0);
431+
assertThat(requests.get(1).getPayload()).isEqualTo("01234567".getBytes(StandardCharsets.UTF_8));
432+
}
433+
434+
@Test
435+
void testChunkRetry_transientFailureExhaustion_surfacesLastError() {
436+
stubStartSession("https://upload.url/chunk-exhaustion");
437+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
438+
.thenReturn(
439+
ApiFutures.immediateFailedFuture(createApiException(503, StatusCode.Code.UNAVAILABLE)));
440+
441+
ResumableUploadFuture<String> future =
442+
callable.futureCall("resource-path", streamOf("hello"), null);
443+
444+
ExecutionException exception = assertThrows(ExecutionException.class, future::get);
445+
assertThat(exception.getCause()).isInstanceOf(ApiException.class);
446+
assertThat(((ApiException) exception.getCause()).getStatusCode().getTransportCode())
447+
.isEqualTo(503);
448+
449+
// Default chunk retry settings has maxAttempts = 5
450+
verify(mockChunkCallable, times(5)).futureCall(any(), any());
451+
}
452+
453+
@Test
454+
void testChunkRetry_cancellationDuringBackoff_deschedulesPendingAttempt() {
455+
SettableApiFuture<ChunkUploadResponse<String>> chunkAttempt0Future = SettableApiFuture.create();
456+
when(mockChunkCallable.futureCall(any(ChunkUploadRequest.class), any()))
457+
.thenReturn(chunkAttempt0Future)
458+
.thenReturn(
459+
ApiFutures.immediateFuture(
460+
ChunkUploadResponse.create(ResumableUploadStatus.FINAL, "should-not-reach")));
461+
462+
ClientContext clientContext =
463+
ClientContext.newBuilder().setDefaultCallContext(callContext).build();
464+
SettableApiFuture<ResumableUploadSession> startSessionFuture = SettableApiFuture.create();
465+
startSessionFuture.set(
466+
ResumableUploadSession.newBuilder()
467+
.setUploadUrl("https://upload.url/cancel-backoff")
468+
.build());
469+
ResumableUploadCallSettings customSettings =
470+
defaultSettings.toBuilder().setChunkSize(8).build();
471+
ResumableUploadFutureImpl<String> sessionFuture =
472+
ResumableUploadFutureImpl.create(
473+
startSessionFuture,
474+
mockChunkCallable,
475+
streamOf("hello"),
476+
customSettings,
477+
clientContext);
478+
479+
// Fail attempt 0 with 503 to schedule backoff
480+
chunkAttempt0Future.setException(createApiException(503, StatusCode.Code.UNAVAILABLE));
481+
482+
// Cancel while backoff is pending
483+
assertThat(sessionFuture.cancel(true)).isTrue();
484+
assertThat(sessionFuture.isCancelled()).isTrue();
485+
assertThrows(CancellationException.class, sessionFuture::get);
486+
487+
// Only attempt 0 occurred; attempt 1 was de-scheduled
488+
verify(mockChunkCallable, times(1)).futureCall(any(), any());
489+
}
490+
491+
private static class HttpStatusStatusCode implements StatusCode {
492+
private final int httpStatus;
493+
private final StatusCode.Code code;
494+
495+
HttpStatusStatusCode(int httpStatus, StatusCode.Code code) {
496+
this.httpStatus = httpStatus;
497+
this.code = code;
498+
}
499+
500+
@Override
501+
public StatusCode.Code getCode() {
502+
return code;
503+
}
504+
505+
@Override
506+
public Integer getTransportCode() {
507+
return httpStatus;
508+
}
509+
}
510+
511+
private static ApiException createApiException(int httpStatus, StatusCode.Code code) {
512+
return ApiExceptionFactory.createException(
513+
"HTTP " + httpStatus, null, new HttpStatusStatusCode(httpStatus, code), false);
514+
}
515+
407516
private void stubStartSession(String uploadUrl) {
408517
when(mockStartCallable.futureCall(any(), any()))
409518
.thenReturn(
@@ -424,11 +533,21 @@ private static void assertChunk(
424533

425534
private static class TrackableStream extends ByteArrayInputStream {
426535
boolean closed = false;
536+
int totalBytesRead = 0;
427537

428538
TrackableStream(String content) {
429539
super(content.getBytes(StandardCharsets.UTF_8));
430540
}
431541

542+
@Override
543+
public int read(byte[] b, int off, int len) {
544+
int read = super.read(b, off, len);
545+
if (read > 0) {
546+
totalBytesRead += read;
547+
}
548+
return read;
549+
}
550+
432551
@Override
433552
public void close() throws IOException {
434553
closed = true;

0 commit comments

Comments
 (0)