feat(gax): add recoverable error query and buffer realignment loop

Introduces a query-and-realign loop when encountering recoverable protocol errors during chunk uploads. Queries the server for the committed offset and adjusts the buffer window before resuming chunk transmission.

Author: whowes

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadCallableImpl.java

@@ -37,6 +37,8 @@
 import com.google.api.core.InternalApi;
 import com.google.api.gax.resumable.ChunkUploadRequest;
 import com.google.api.gax.resumable.ChunkUploadResponse;
+import com.google.api.gax.resumable.QueryStatusRequest;
+import com.google.api.gax.resumable.QueryStatusResponse;
 import com.google.api.gax.resumable.ResumableUploadClient;
 import com.google.api.gax.resumable.ResumableUploadSession;
 import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
@@ -74,6 +76,8 @@ public class ResumableUploadCallableImpl<RequestT, ResponseT>
   private final ClientContext clientContext;
   private final UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>>
       retryingUploadChunkCallable;
+  private final UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>>
+      retryingQueryCallable;
 
   public ResumableUploadCallableImpl(
       ResumableUploadClient<RequestT, ResponseT> client,
@@ -89,6 +93,8 @@ public ResumableUploadCallableImpl(
             .build();
     this.retryingUploadChunkCallable =
         createRetryingCallable(client.uploadChunkCallable(), ResumableUploadCommand.UPLOAD);
+    this.retryingQueryCallable =
+        createRetryingCallable(client.queryStatusCallable(), ResumableUploadCommand.QUERY);
   }
 
   @Override
@@ -110,7 +116,12 @@ public ResumableUploadFuture<ResponseT> futureCall(
     }
 
     return ResumableUploadFutureImpl.create(
-        startFuture, retryingUploadChunkCallable, payload, effectiveSettings, clientContext);
+        startFuture,
+        retryingUploadChunkCallable,
+        retryingQueryCallable,
+        payload,
+        effectiveSettings,
+        clientContext);
   }
 
   @Override

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadChunkCoordinator.java

@@ -38,7 +38,10 @@
 import com.google.api.core.SettableApiFuture;
 import com.google.api.gax.resumable.ChunkUploadRequest;
 import com.google.api.gax.resumable.ChunkUploadResponse;
+import com.google.api.gax.resumable.QueryStatusRequest;
+import com.google.api.gax.resumable.QueryStatusResponse;
 import com.google.api.gax.resumable.ResumableUploadStatus;
+import com.google.api.gax.rpc.ResumableUploadErrorClassifier.Category;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.io.IOException;
 import java.io.InputStream;
@@ -50,6 +53,9 @@
 /**
  * Coordinates chunk transmission steps of a resumable upload session.
  *
+ * <p>Expects {@code uploadChunkCallable} and {@code queryStatusCallable} to be pre-wrapped in
+ * retrying callables that handle transient errors.
+ *
  * @param <ResponseT> the type of the final response message returned once the upload completes
  */
 @InternalApi
@@ -65,6 +71,8 @@ final class ResumableUploadChunkCoordinator<ResponseT> {
 
   private final UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>>
       uploadChunkCallable;
+  private final UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>>
+      queryStatusCallable;
   private final String uploadUrl;
   private final RewindableStreamBuffer buffer;
   private final ApiCallContext callContext;
@@ -73,12 +81,15 @@ final class ResumableUploadChunkCoordinator<ResponseT> {
 
   ResumableUploadChunkCoordinator(
       UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
+      UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>> queryStatusCallable,
       String uploadUrl,
       InputStream payload,
       int chunkSize,
       ClientContext clientContext) {
     this.uploadChunkCallable =
         checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
+    this.queryStatusCallable =
+        checkNotNull(queryStatusCallable, "queryStatusCallable must not be null");
     this.uploadUrl = checkNotNull(uploadUrl, "uploadUrl must not be null");
     checkNotNull(payload, "payload must not be null");
     checkNotNull(clientContext, "clientContext must not be null");
@@ -113,48 +124,24 @@ private void transmitChunk(long currentOffset) {
       return;
     }
 
-    // Determine if this is the final chunk and build the chunk request.
-    ChunkUploadRequest chunkRequest =
-        ChunkUploadRequest.newBuilder()
-            .setUploadUrl(uploadUrl)
-            .setPayload(buffer.getBuffer())
-            .setPayloadLength(buffer.getPayloadLength())
-            .setOffset(buffer.getBufferBaseOffset())
-            .setFinal(buffer.isFinal())
-            .build();
+    dispatchCurrentChunk();
+  }
 
+  private void dispatchCurrentChunk() {
     // Dispatch the chunk upload call and register the in-flight future for cancellation.
-    long chunkLength = chunkRequest.getPayloadLength();
-    boolean isFinal = chunkRequest.isFinal();
     try {
-      ApiFuture<ChunkUploadResponse<ResponseT>> chunkFuture =
-          uploadChunkCallable.futureCall(chunkRequest, callContext);
+      ApiFuture<ChunkUploadResponse<ResponseT>> chunkFuture = executeChunkWithRecovery();
       this.currentChunkFuture = chunkFuture;
       if (result.isCancelled()) {
         chunkFuture.cancel(true);
         return;
       }
-
       ApiFutures.addCallback(
           chunkFuture,
           new ApiFutureCallback<ChunkUploadResponse<ResponseT>>() {
             @Override
             public void onSuccess(ChunkUploadResponse<ResponseT> response) {
-              if (result.isDone()) {
-                return;
-              }
-              long nextOffset = currentOffset + chunkLength;
-              if (response.getUploadStatus() == ResumableUploadStatus.FINAL) {
-                result.set(response.getResponse());
-              } else if (isFinal) {
-                result.setException(
-                    new IllegalStateException(
-                        "Upload stream ended and final chunk was transmitted, but server returned"
-                            + " incomplete status for upload URL: "
-                            + uploadUrl));
-              } else {
-                chunkExecutor.execute(() -> transmitChunk(nextOffset));
-              }
+              chunkExecutor.execute(() -> onChunkUploaded(response));
             }
 
             @Override
@@ -170,4 +157,104 @@ public void onFailure(Throwable t) {
       result.setException(t);
     }
   }
+
+  private ApiFuture<ChunkUploadResponse<ResponseT>> executeChunkWithRecovery() {
+    ApiFuture<ChunkUploadResponse<ResponseT>> transmitted =
+        ApiFutures.catchingAsync(
+            uploadChunkCallable.futureCall(buildCurrentChunkRequest(), callContext),
+            Throwable.class,
+            t -> {
+              Category category =
+                  ResumableUploadErrorClassifier.classify(t, ResumableUploadCommand.UPLOAD);
+              if (category == Category.RECOVERABLE) {
+                return recover();
+              }
+              // Category.TRANSIENT errors reaching here have already exhausted their retry budget
+              // in the underlying RetryingCallable and become fatal per protocol specification.
+              return ApiFutures.immediateFailedFuture(t);
+            },
+            chunkExecutor);
+    return ApiFutures.transformAsync(
+        transmitted,
+        response -> {
+          if (response.getUploadStatus() == ResumableUploadStatus.UNKNOWN) {
+            return recover();
+          }
+          return ApiFutures.immediateFuture(response);
+        },
+        chunkExecutor);
+  }
+
+  private ApiFuture<ChunkUploadResponse<ResponseT>> recover() {
+    return ApiFutures.transformAsync(
+        queryStatusCallable.futureCall(QueryStatusRequest.create(uploadUrl), callContext),
+        this::resumeFrom,
+        chunkExecutor);
+  }
+
+  private ApiFuture<ChunkUploadResponse<ResponseT>> resumeFrom(
+      QueryStatusResponse<ResponseT> queryResponse) throws IOException {
+    if (queryResponse.getUploadStatus() == ResumableUploadStatus.UNKNOWN) {
+      throw protocolViolation(
+          "Query status response missing X-Goog-Upload-Status header for upload URL: " + uploadUrl);
+    }
+    if (queryResponse.getUploadStatus() == ResumableUploadStatus.FINAL) {
+      return ApiFutures.immediateFuture(
+          ChunkUploadResponse.create(ResumableUploadStatus.FINAL, queryResponse.getResponse()));
+    }
+    Long committedOffset = queryResponse.getCommittedOffset();
+    if (committedOffset == null) {
+      throw protocolViolation(
+          "Incomplete query status response did not include a committed offset for upload URL: "
+              + uploadUrl);
+    }
+    buffer.realignTo(committedOffset);
+    return executeChunkWithRecovery();
+  }
+
+  private void onChunkUploaded(ChunkUploadResponse<ResponseT> response) {
+    if (result.isDone()) {
+      return;
+    }
+    long nextOffset = buffer.getBufferBaseOffset() + buffer.getPayloadLength();
+    if (response.getUploadStatus() == ResumableUploadStatus.FINAL) {
+      result.set(response.getResponse());
+    } else if (buffer.isFinal()) {
+      result.setException(
+          new IllegalStateException(
+              "Upload stream ended and final chunk was transmitted, but server returned"
+                  + " incomplete status for upload URL: "
+                  + uploadUrl));
+    } else {
+      chunkExecutor.execute(() -> transmitChunk(nextOffset));
+    }
+  }
+
+  private ChunkUploadRequest buildCurrentChunkRequest() {
+    // Determine if this is the final chunk and build the chunk request.
+    return ChunkUploadRequest.newBuilder()
+        .setUploadUrl(uploadUrl)
+        .setPayload(buffer.getBuffer())
+        .setPayloadLength(buffer.getPayloadLength())
+        .setOffset(buffer.getBufferBaseOffset())
+        .setFinal(buffer.isFinal())
+        .build();
+  }
+
+  private static final StatusCode FAILED_PRECONDITION_STATUS_CODE =
+      new StatusCode() {
+        @Override
+        public StatusCode.Code getCode() {
+          return StatusCode.Code.FAILED_PRECONDITION;
+        }
+
+        @Override
+        public @Nullable Object getTransportCode() {
+          return null;
+        }
+      };
+
+  private static FailedPreconditionException protocolViolation(String message) {
+    return new FailedPreconditionException(message, null, FAILED_PRECONDITION_STATUS_CODE, false);
+  }
 }

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ResumableUploadFutureImpl.java

@@ -38,6 +38,8 @@
 import com.google.api.core.SettableApiFuture;
 import com.google.api.gax.resumable.ChunkUploadRequest;
 import com.google.api.gax.resumable.ChunkUploadResponse;
+import com.google.api.gax.resumable.QueryStatusRequest;
+import com.google.api.gax.resumable.QueryStatusResponse;
 import com.google.api.gax.resumable.ResumableUploadSession;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.errorprone.annotations.concurrent.GuardedBy;
@@ -65,6 +67,8 @@ final class ResumableUploadFutureImpl<ResponseT> implements ResumableUploadFutur
   private final ApiFuture<ResumableUploadSession> startFuture;
   private final UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>>
       uploadChunkCallable;
+  private final UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>>
+      queryStatusCallable;
   private final InputStream payload;
   private final ResumableUploadCallSettings settings;
   private final ClientContext clientContext;
@@ -85,12 +89,18 @@ final class ResumableUploadFutureImpl<ResponseT> implements ResumableUploadFutur
   static <ResponseT> ResumableUploadFutureImpl<ResponseT> create(
       ApiFuture<ResumableUploadSession> startFuture,
       UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
+      UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>> queryStatusCallable,
       InputStream payload,
       ResumableUploadCallSettings settings,
       ClientContext clientContext) {
     ResumableUploadFutureImpl<ResponseT> future =
         new ResumableUploadFutureImpl<>(
-            startFuture, uploadChunkCallable, payload, settings, clientContext);
+            startFuture,
+            uploadChunkCallable,
+            queryStatusCallable,
+            payload,
+            settings,
+            clientContext);
     try {
       future.start();
     } catch (Throwable t) {
@@ -102,12 +112,15 @@ static <ResponseT> ResumableUploadFutureImpl<ResponseT> create(
   private ResumableUploadFutureImpl(
       ApiFuture<ResumableUploadSession> startFuture,
       UnaryCallable<ChunkUploadRequest, ChunkUploadResponse<ResponseT>> uploadChunkCallable,
+      UnaryCallable<QueryStatusRequest, QueryStatusResponse<ResponseT>> queryStatusCallable,
       InputStream payload,
       ResumableUploadCallSettings settings,
       ClientContext clientContext) {
     this.startFuture = checkNotNull(startFuture, "startFuture must not be null");
     this.uploadChunkCallable =
         checkNotNull(uploadChunkCallable, "uploadChunkCallable must not be null");
+    this.queryStatusCallable =
+        checkNotNull(queryStatusCallable, "queryStatusCallable must not be null");
     this.payload = checkNotNull(payload, "payload must not be null");
     this.settings = checkNotNull(settings, "settings must not be null");
     checkArgument(settings.getChunkSize() > 0, "chunkSize must be > 0");
@@ -128,6 +141,7 @@ public void onSuccess(ResumableUploadSession session) {
             ResumableUploadChunkCoordinator<ResponseT> coordinator =
                 new ResumableUploadChunkCoordinator<>(
                     uploadChunkCallable,
+                    queryStatusCallable,
                     uploadSessionUrl,
                     payload,
                     settings.getChunkSize(),

sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/CallableTest.java

@@ -216,6 +216,8 @@ void testResumableUploadCallable() {
         mock(ResumableUploadClient.class, Mockito.withSettings().withoutAnnotations());
     when(uploadClient.uploadChunkCallable())
         .thenReturn(mock(UnaryCallable.class, Mockito.withSettings().withoutAnnotations()));
+    when(uploadClient.queryStatusCallable())
+        .thenReturn(mock(UnaryCallable.class, Mockito.withSettings().withoutAnnotations()));
     ResumableUploadCallSettings settings =
         ResumableUploadCallSettings.newBuilder().setChunkSize(1024).build();