Skip to content

Commit 5280280

Browse files
committed
feat(gax): enforce global timeout for resumable uploads
Enforces ResumableUploadCallSettings.getGlobalTimeout() in ResumableUploadChunkCoordinator across the upload lifecycle. Cancels in-flight RPCs and completes the future with DeadlineExceededException when the deadline is exceeded.
1 parent f9ca3dc commit 5280280

6 files changed

Lines changed: 426 additions & 45 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallSettings.java‎

Lines changed: 51 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,31 @@
4545
@NullMarked
4646
public abstract class ResumableUploadCallSettings {
4747
private static final int DEFAULT_CHUNK_SIZE = 8 * 1024 * 1024; // 8 MB
48+
// Matches Ruby google-apis-core RequestOptions.default.max_elapsed_time = 900s (CL-R9).
49+
private static final Duration DEFAULT_GLOBAL_TIMEOUT = Duration.ofMinutes(15);
50+
51+
abstract @Nullable Integer chunkSizeOption();
52+
53+
abstract @Nullable Duration globalTimeoutOption();
4854

4955
/** Returns the configured chunk size in bytes (defaults to 8 MB / 8,388,608 bytes). */
50-
public abstract int getChunkSize();
56+
public int getChunkSize() {
57+
Integer size = chunkSizeOption();
58+
return size != null ? size : DEFAULT_CHUNK_SIZE;
59+
}
5160

5261
/**
53-
* Returns the global upload timeout governing the entire upload duration, or {@code null} if
54-
* disabled.
62+
* Returns the global upload timeout governing the entire upload duration (defaults to 15
63+
* minutes).
5564
*/
56-
public abstract @Nullable Duration getGlobalTimeout();
65+
public Duration getGlobalTimeout() {
66+
Duration timeout = globalTimeoutOption();
67+
return timeout != null ? timeout : DEFAULT_GLOBAL_TIMEOUT;
68+
}
5769

5870
/**
59-
* Merges another {@code ResumableUploadCallSettings} instance with this one. Fields set in {@code
60-
* other} override fields in this instance.
71+
* Merges another {@code ResumableUploadCallSettings} instance with this one. Fields explicitly
72+
* set in {@code other} override fields in this instance.
6173
*
6274
* @param other settings to overlay; may be {@code null}
6375
* @return a new, resolved {@code ResumableUploadCallSettings} instance
@@ -67,40 +79,60 @@ public ResumableUploadCallSettings merge(@Nullable ResumableUploadCallSettings o
6779
return this;
6880
}
6981
Builder builder = toBuilder();
70-
if (other.getChunkSize() > 0) {
71-
builder.setChunkSize(other.getChunkSize());
82+
if (other.chunkSizeOption() != null) {
83+
builder.setChunkSize(other.chunkSizeOption());
7284
}
73-
if (other.getGlobalTimeout() != null) {
74-
builder.setGlobalTimeout(other.getGlobalTimeout());
85+
if (other.globalTimeoutOption() != null) {
86+
builder.setGlobalTimeout(other.globalTimeoutOption());
7587
}
7688
return builder.build();
7789
}
7890

7991
public abstract Builder toBuilder();
8092

8193
public static Builder newBuilder() {
82-
return new AutoValue_ResumableUploadCallSettings.Builder().setChunkSize(DEFAULT_CHUNK_SIZE);
94+
return new AutoValue_ResumableUploadCallSettings.Builder();
8395
}
8496

8597
/** Builder for {@link ResumableUploadCallSettings}. */
8698
@AutoValue.Builder
8799
public abstract static class Builder {
88-
public abstract Builder setChunkSize(int chunkSize);
100+
abstract Builder setChunkSizeOption(@Nullable Integer chunkSize);
89101

90-
public abstract int getChunkSize();
102+
abstract @Nullable Integer chunkSizeOption();
91103

92-
public abstract Builder setGlobalTimeout(@Nullable Duration globalTimeout);
104+
public Builder setChunkSize(int chunkSize) {
105+
return setChunkSizeOption(chunkSize);
106+
}
107+
108+
public int getChunkSize() {
109+
Integer size = chunkSizeOption();
110+
return size != null ? size : DEFAULT_CHUNK_SIZE;
111+
}
93112

94-
public abstract @Nullable Duration getGlobalTimeout();
113+
abstract Builder setGlobalTimeoutOption(@Nullable Duration globalTimeout);
114+
115+
abstract @Nullable Duration globalTimeoutOption();
116+
117+
public Builder setGlobalTimeout(@Nullable Duration globalTimeout) {
118+
return setGlobalTimeoutOption(globalTimeout);
119+
}
120+
121+
public @Nullable Duration getGlobalTimeout() {
122+
return globalTimeoutOption();
123+
}
95124

96125
abstract ResumableUploadCallSettings autoBuild();
97126

98127
public ResumableUploadCallSettings build() {
99-
Preconditions.checkArgument(getChunkSize() > 0, "chunkSize must be > 0");
100-
if (getGlobalTimeout() != null) {
128+
Integer size = chunkSizeOption();
129+
if (size != null) {
130+
Preconditions.checkArgument(size > 0, "chunkSize must be > 0");
131+
}
132+
Duration timeout = globalTimeoutOption();
133+
if (timeout != null) {
101134
Preconditions.checkArgument(
102-
!getGlobalTimeout().isNegative() && !getGlobalTimeout().isZero(),
103-
"globalTimeout must be positive");
135+
!timeout.isNegative() && !timeout.isZero(), "globalTimeout must be positive");
104136
}
105137
return autoBuild();
106138
}

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public ResumableUploadFuture<ResponseT> futureCall(
119119
retryingQueryCallable,
120120
payload,
121121
effectiveSettings,
122-
clientContext.getDefaultCallContext());
122+
clientContext);
123123
}
124124

125125
@Override

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java‎

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,12 @@
4141
import com.google.api.gax.resumable.QueryStatusRequest;
4242
import com.google.api.gax.resumable.QueryStatusResponse;
4343
import com.google.api.gax.resumable.ResumableUploadStatus;
44+
import com.google.api.gax.retrying.RetrySettings;
4445
import com.google.api.gax.rpc.ResumableUploadErrorClassifier.Category;
4546
import com.google.common.util.concurrent.MoreExecutors;
4647
import java.io.IOException;
4748
import java.io.InputStream;
49+
import java.time.Duration;
4850
import java.util.concurrent.CancellationException;
4951
import java.util.concurrent.Executor;
5052
import org.jspecify.annotations.NullMarked;
@@ -72,25 +74,31 @@ final class ResumableUploadChunkCoordinator<ResponseT> {
7274
queryStatusCallable;
7375
private final String uploadUrl;
7476
private final RewindableStreamBuffer buffer;
77+
private final ResumableUploadCallSettings settings;
78+
private final long deadlineNanos;
7579
private final ApiCallContext callContext;
80+
private final ClientContext clientContext;
7681
private final SettableApiFuture<ResponseT> result = SettableApiFuture.create();
7782
private volatile @Nullable ApiFuture<?> currentChunkFuture;
7883

7984
ResumableUploadChunkCoordinator(
8085
UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
8186
UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>> queryStatusCallable,
8287
String uploadUrl,
83-
InputStream payload,
84-
int chunkSize,
85-
ApiCallContext callContext) {
88+
ResumableUploadCallSettings settings,
89+
long deadlineNanos,
90+
ClientContext clientContext) {
8691
this.uploadChunkCallable =
8792
checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
8893
this.queryStatusCallable =
8994
checkNotNull(queryStatusCallable, "queryStatusCallable must not be null");
9095
this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null");
9196
checkNotNull(payload, "payload must not be null");
92-
this.callContext = checkNotNull(callContext, "callContext must not be null");
93-
this.buffer = new RewindableStreamBuffer(payload, chunkSize, uploadUrl);
97+
this.settings = checkNotNull(settings, "settings must not be null");
98+
this.deadlineNanos = deadlineNanos;
99+
this.clientContext = checkNotNull(clientContext, "clientContext must not be null");
100+
this.callContext = clientContext.getDefaultCallContext();
101+
this.buffer = new RewindableStreamBuffer(payload, settings.getChunkSize(), uploadUrl);
94102
}
95103

96104
ApiFuture<ResponseT> start() {
@@ -154,7 +162,7 @@ public void onFailure(Throwable t) {
154162
private ApiFuture<ChunkUploadResponse<ResponseT>> executeChunkWithRecovery() {
155163
ApiFuture<ChunkUploadResponse<ResponseT>> transmitted =
156164
ApiFutures.catchingAsync(
157-
uploadChunkCallable.futureCall(buildCurrentChunkRequest(), callContext),
165+
uploadChunkCallable.futureCall(buildCurrentChunkRequest(), chunkCallContext()),
158166
Throwable.class,
159167
t -> {
160168
Category category =
@@ -180,7 +188,7 @@ private ApiFuture<ChunkUploadResponse<ResponseT>> executeChunkWithRecovery() {
180188

181189
private ApiFuture<ChunkUploadResponse<ResponseT>> recover() {
182190
return ApiFutures.transformAsync(
183-
queryStatusCallable.futureCall(QueryStatusRequest.create(uploadUrl), callContext),
191+
queryStatusCallable.futureCall(QueryStatusRequest.create(uploadUrl), queryCallContext()),
184192
this::resumeFrom,
185193
chunkExecutor);
186194
}
@@ -247,6 +255,60 @@ public StatusCode.Code getCode() {
247255
}
248256
};
249257

258+
/**
259+
* Returns the call context for a chunk upload, bounding its retry cycle and each of its attempts
260+
* to the chunk's local deadline: at most half the global timeout, never past the global deadline.
261+
*/
262+
private ApiCallContext chunkCallContext() {
263+
Duration remaining = remainingGlobalTimeout();
264+
if (remaining == null) {
265+
return callContext;
266+
}
267+
Duration deadline = min(settings.getGlobalTimeout().dividedBy(2), remaining);
268+
return boundedCallContext(
269+
deadline,
270+
callContext.getRetrySettings().toBuilder()
271+
.setInitialRpcTimeoutDuration(deadline)
272+
.setRpcTimeoutMultiplier(1.0)
273+
.setMaxRpcTimeoutDuration(deadline));
274+
}
275+
276+
/** Returns the call context for a status query, bounded by the remaining global timeout. */
277+
private ApiCallContext queryCallContext() {
278+
Duration remaining = remainingGlobalTimeout();
279+
if (remaining == null) {
280+
return callContext;
281+
}
282+
RetrySettings baseSettings = callContext.getRetrySettings();
283+
return boundedCallContext(
284+
remaining,
285+
baseSettings.toBuilder()
286+
.setInitialRpcTimeoutDuration(
287+
min(baseSettings.getInitialRpcTimeoutDuration(), remaining))
288+
.setMaxRpcTimeoutDuration(min(baseSettings.getMaxRpcTimeoutDuration(), remaining)));
289+
}
290+
291+
/**
292+
* Returns the time left before the global upload deadline, or null if deadlines cannot be derived
293+
* because no global timeout or no base retry settings were configured.
294+
*/
295+
private @Nullable Duration remainingGlobalTimeout() {
296+
if (settings.getGlobalTimeout() == null || callContext.getRetrySettings() == null) {
297+
return null;
298+
}
299+
return Duration.ofNanos(Math.max(1L, deadlineNanos - clientContext.getClock().nanoTime()));
300+
}
301+
302+
private ApiCallContext boundedCallContext(Duration deadline, RetrySettings.Builder settings) {
303+
return callContext
304+
.withRetrySettings(settings.setTotalTimeoutDuration(deadline).build())
305+
.withTimeoutDuration(deadline);
306+
}
307+
308+
private static Duration min(Duration left, Duration right) {
309+
return left.compareTo(right) <= 0 ? left : right;
310+
}
311+
250312
private static FailedPreconditionException protocolViolation(String message) {
251313
return new FailedPreconditionException(message, null, FAILED_PRECONDITION_STATUS_CODE, false);
252314
}

0 commit comments

Comments
 (0)