Skip to content

Commit e4a1348

Browse files
committed
feat(bigquery): support mid-flight slow-path transition for incomplete Arrow queries
1 parent 2964b97 commit e4a1348

2 files changed

Lines changed: 201 additions & 26 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2558,10 +2558,23 @@ && getOptions().getOpenTelemetryTracer() != null) {
25582558
throw new BigQueryException(bigQueryErrors);
25592559
}
25602560

2561-
// If query is incomplete, Arrow format for slow query path is not yet supported.
2561+
// If the query is incomplete (took longer than the fast-path timeout), transition
2562+
// to the slow path: retrieve the created job, wait for completion, and read the
2563+
// results using the Storage Read API in Arrow format.
25622564
if (!Boolean.TRUE.equals(results.getJobComplete())) {
2563-
throw new UnsupportedOperationException(
2564-
"Arrow results format for slow query path execution is not yet supported.");
2565+
if (results.getJobReference() == null) {
2566+
throw new BigQueryException(
2567+
0, "Query is incomplete, but no job reference was returned to await completion.");
2568+
}
2569+
JobId jobId = JobId.fromPb(results.getJobReference());
2570+
Job job = getJob(jobId, options);
2571+
if (job == null) {
2572+
throw new BigQueryException(0, "Job no longer exists or could not be retrieved: " + jobId);
2573+
}
2574+
Job completedJob = job.waitFor();
2575+
Long maxResults =
2576+
content.getMaxResults() != null ? content.getMaxResults().longValue() : null;
2577+
return readArrowTableResultFromJob(completedJob, maxResults, null, options);
25652578
}
25662579

25672580
// If the query completed but the Arrow schema is missing from the response, fail fast.
@@ -3018,24 +3031,18 @@ private ArrowQueryResult createArrowQueryResultFromTable(
30183031
}
30193032

30203033
/**
3021-
* Executes a slow-path query job using {@code jobs.insert}, awaits its completion, and streams
3022-
* the result rows via the BigQuery Storage Read API in Arrow format, wrapping the decoded rows in
3023-
* a {@link TableResult}.
3034+
* Reads query result rows from a completed query job's destination table using the Storage Read
3035+
* API in Arrow format, returning a populated {@link TableResult}.
30243036
*
3025-
* @param jobId the job ID, or {@code null}
3026-
* @param configuration the query job configuration
3037+
* @param completedJob the completed query job
3038+
* @param maxResults maximum results requested, or {@code null}
3039+
* @param fallbackDestinationTable fallback destination table if not present on the job
30273040
* @param options query job options
3028-
* @return a {@link TableResult} containing the decoded rows and job execution metadata
3029-
* @throws InterruptedException if interrupted while awaiting job completion
3030-
* @throws BigQueryException if job execution or ReadSession creation fails
3041+
* @return a {@link TableResult} containing decoded rows and execution metadata
3042+
* @throws BigQueryException if job failed or ReadSession creation fails
30313043
*/
3032-
private TableResult queryFallbackArrow(
3033-
JobId jobId, QueryJobConfiguration configuration, JobOption... options)
3034-
throws InterruptedException {
3035-
// Submit the query job via jobs.insert and poll until completion.
3036-
Job job = create(JobInfo.of(jobId, configuration), options);
3037-
Job completedJob = job.waitFor();
3038-
3044+
private TableResult readArrowTableResultFromJob(
3045+
Job completedJob, Long maxResults, TableId fallbackDestinationTable, JobOption... options) {
30393046
if (completedJob == null) {
30403047
throw new BigQueryException(0, "Job no longer exists or could not be retrieved.");
30413048
}
@@ -3044,17 +3051,17 @@ private TableResult queryFallbackArrow(
30443051
throw new BigQueryException(Collections.singletonList(completedJob.getStatus().getError()));
30453052
}
30463053

3047-
// Resolve the query's destination table where the completed job wrote its results.
3054+
// Resolve the destination table containing the query results.
30483055
TableId destinationTable = null;
30493056
if (completedJob.getConfiguration() instanceof QueryJobConfiguration) {
30503057
destinationTable =
30513058
((QueryJobConfiguration) completedJob.getConfiguration()).getDestinationTable();
30523059
}
30533060
if (destinationTable == null) {
3054-
destinationTable = configuration.getDestinationTable();
3061+
destinationTable = fallbackDestinationTable;
30553062
}
30563063
if (destinationTable == null) {
3057-
throw new BigQueryException(0, "Unable to resolve destination table for fallback query");
3064+
throw new BigQueryException(0, "Unable to resolve destination table for query job");
30583065
}
30593066

30603067
// Extract query execution statistics from the completed job metadata.
@@ -3095,7 +3102,7 @@ private TableResult queryFallbackArrow(
30953102
try {
30963103
readSession = client.createReadSession(request);
30973104
} catch (Exception e) {
3098-
throw new BigQueryException(0, "Failed to create ReadSession for fallback query", e);
3105+
throw new BigQueryException(0, "Failed to create ReadSession for query job", e);
30993106
}
31003107

31013108
// Deserialize the Arrow schema and convert to BigQuery Schema for TableResult metadata.
@@ -3145,7 +3152,7 @@ private TableResult queryFallbackArrow(
31453152
arrowSchemaPojo,
31463153
getOptions(),
31473154
0L,
3148-
configuration.getMaxResults(),
3155+
maxResults,
31493156
optionMap(options));
31503157

31513158
Page<FieldValueList> firstPage = pageFetcher.getNextPage();
@@ -3179,6 +3186,28 @@ private TableResult queryFallbackArrow(
31793186
.build();
31803187
}
31813188

3189+
/**
3190+
* Executes a slow-path query job using {@code jobs.insert}, awaits its completion, and streams
3191+
* the result rows via the BigQuery Storage Read API in Arrow format, wrapping the decoded rows in
3192+
* a {@link TableResult}.
3193+
*
3194+
* @param jobId the job ID, or {@code null}
3195+
* @param configuration the query job configuration
3196+
* @param options query job options
3197+
* @return a {@link TableResult} containing the decoded rows and job execution metadata
3198+
* @throws InterruptedException if interrupted while awaiting job completion
3199+
* @throws BigQueryException if job execution or ReadSession creation fails
3200+
*/
3201+
private TableResult queryFallbackArrow(
3202+
JobId jobId, QueryJobConfiguration configuration, JobOption... options)
3203+
throws InterruptedException {
3204+
// Submit the query job via jobs.insert and poll until completion.
3205+
Job job = create(JobInfo.of(jobId, configuration), options);
3206+
Job completedJob = job.waitFor();
3207+
return readArrowTableResultFromJob(
3208+
completedJob, configuration.getMaxResults(), configuration.getDestinationTable(), options);
3209+
}
3210+
31823211
@Override
31833212
public QueryResponse getQueryResults(JobId jobId, QueryResultsOption... options) {
31843213
Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java‎

Lines changed: 149 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3410,20 +3410,166 @@ void testQueryWithArrowFormatIncompleteJob() throws Exception {
34103410
.setJobComplete(false)
34113411
.setJobReference(queryJob.toPb());
34123412

3413+
com.google.api.services.bigquery.model.JobStatus jobStatus =
3414+
new com.google.api.services.bigquery.model.JobStatus().setState("DONE");
3415+
3416+
com.google.api.services.bigquery.model.Job jobResponsePb =
3417+
new com.google.api.services.bigquery.model.Job()
3418+
.setJobReference(queryJob.toPb())
3419+
.setStatus(jobStatus)
3420+
.setConfiguration(
3421+
new com.google.api.services.bigquery.model.JobConfiguration()
3422+
.setQuery(
3423+
new JobConfigurationQuery()
3424+
.setQuery("SELECT id FROM test")
3425+
.setDestinationTable(TABLE_ID.toPb())))
3426+
.setStatistics(
3427+
new com.google.api.services.bigquery.model.JobStatistics()
3428+
.setTotalSlotMs(50L)
3429+
.setQuery(
3430+
new com.google.api.services.bigquery.model.JobStatistics2()
3431+
.setStatementType("SELECT")
3432+
.setTotalBytesBilled(100L)
3433+
.setTotalBytesProcessed(200L)));
3434+
34133435
when(bigqueryRpcMock.queryRpcSkipExceptionTranslation(eq(PROJECT), any(QueryRequest.class)))
34143436
.thenReturn(queryResponsePb);
3437+
when(bigqueryRpcMock.getJobSkipExceptionTranslation(eq(PROJECT), eq(JOB), any(), any()))
3438+
.thenReturn(jobResponsePb);
3439+
when(bigqueryRpcMock.getQueryResultsSkipExceptionTranslation(
3440+
eq(PROJECT), eq(JOB), any(), any()))
3441+
.thenReturn(
3442+
new GetQueryResultsResponse().setJobComplete(true).setJobReference(queryJob.toPb()));
3443+
3444+
org.apache.arrow.vector.types.pojo.Schema arrowSchema =
3445+
new org.apache.arrow.vector.types.pojo.Schema(
3446+
ImmutableList.of(
3447+
org.apache.arrow.vector.types.pojo.Field.nullable(
3448+
"id", new ArrowType.Int(64, true))));
3449+
3450+
byte[] schemaBytes;
3451+
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
3452+
MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), arrowSchema);
3453+
schemaBytes = out.toByteArray();
3454+
}
3455+
3456+
byte[] batchBytes;
3457+
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
3458+
BigIntVector idVector = new BigIntVector("id", allocator);
3459+
idVector.allocateNew(1);
3460+
idVector.set(0, 42L);
3461+
idVector.setValueCount(1);
3462+
try (VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(idVector))) {
3463+
VectorUnloader unloader = new VectorUnloader(root);
3464+
try (ArrowRecordBatch recordBatch = unloader.getRecordBatch();
3465+
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
3466+
WriteChannel channel = new WriteChannel(Channels.newChannel(out));
3467+
MessageSerializer.serialize(channel, recordBatch);
3468+
batchBytes = out.toByteArray();
3469+
}
3470+
} finally {
3471+
idVector.close();
3472+
}
3473+
}
3474+
3475+
com.google.cloud.bigquery.storage.v1.ArrowRecordBatch protoBatch =
3476+
com.google.cloud.bigquery.storage.v1.ArrowRecordBatch.newBuilder()
3477+
.setSerializedRecordBatch(ByteString.copyFrom(batchBytes))
3478+
.build();
3479+
ReadRowsResponse streamResponse =
3480+
ReadRowsResponse.newBuilder().setArrowRecordBatch(protoBatch).build();
3481+
3482+
@SuppressWarnings("unchecked")
3483+
ServerStreamingCallable<ReadRowsRequest, ReadRowsResponse> mockCallable =
3484+
mock(ServerStreamingCallable.class, withSettings().withoutAnnotations());
3485+
@SuppressWarnings("unchecked")
3486+
ServerStream<ReadRowsResponse> mockServerStream =
3487+
mock(ServerStream.class, withSettings().withoutAnnotations());
3488+
when(mockCallable.call(any(ReadRowsRequest.class))).thenReturn(mockServerStream);
3489+
when(mockServerStream.iterator()).thenReturn(ImmutableList.of(streamResponse).iterator());
3490+
3491+
ReadSession readSession =
3492+
ReadSession.newBuilder()
3493+
.setName("projects/" + PROJECT + "/locations/" + LOCATION + "/sessions/session-1")
3494+
.setArrowSchema(
3495+
com.google.cloud.bigquery.storage.v1.ArrowSchema.newBuilder()
3496+
.setSerializedSchema(ByteString.copyFrom(schemaBytes)))
3497+
.addStreams(ReadStream.newBuilder().setName("stream-1"))
3498+
.build();
3499+
3500+
BigQueryReadClient mockReadClient =
3501+
mock(BigQueryReadClient.class, withSettings().withoutAnnotations());
3502+
when(mockReadClient.createReadSession(any(CreateReadSessionRequest.class)))
3503+
.thenReturn(readSession);
3504+
when(mockReadClient.readRowsCallable()).thenReturn(mockCallable);
34153505

34163506
bigquery = options.getService();
3507+
((BigQueryImpl) bigquery).setBigQueryReadClient(mockReadClient);
34173508

34183509
QueryJobConfiguration config =
34193510
QueryJobConfiguration.newBuilder("SELECT id FROM test")
34203511
.setQueryResultsFormat(QueryResultsFormat.ARROW)
34213512
.build();
3422-
UnsupportedOperationException e =
3423-
assertThrows(UnsupportedOperationException.class, () -> bigquery.query(config));
3513+
3514+
TableResult result = bigquery.query(config);
3515+
3516+
assertNotNull(result);
3517+
assertEquals(1, Iterables.size(result.iterateAll()));
3518+
FieldValueList row = result.iterateAll().iterator().next();
3519+
assertEquals(42L, row.get("id").getLongValue());
3520+
assertEquals(50L, result.getTotalSlotMs().longValue());
3521+
assertEquals(100L, result.getTotalBytesBilled().longValue());
3522+
assertEquals(200L, result.getTotalBytesProcessed().longValue());
3523+
assertEquals(StatementType.SELECT, result.getStatementType());
3524+
}
3525+
3526+
@Test
3527+
void testQueryWithArrowFormatIncompleteJobMissingJobReference() throws Exception {
3528+
com.google.api.services.bigquery.model.QueryResponse queryResponsePb =
3529+
new com.google.api.services.bigquery.model.QueryResponse()
3530+
.setQueryId("q-arrow-incomplete-no-job")
3531+
.setJobComplete(false);
3532+
3533+
when(bigqueryRpcMock.queryRpcSkipExceptionTranslation(eq(PROJECT), any(QueryRequest.class)))
3534+
.thenReturn(queryResponsePb);
3535+
3536+
bigquery = options.getService();
3537+
3538+
QueryJobConfiguration config =
3539+
QueryJobConfiguration.newBuilder("SELECT id FROM test")
3540+
.setQueryResultsFormat(QueryResultsFormat.ARROW)
3541+
.build();
3542+
3543+
BigQueryException e = assertThrows(BigQueryException.class, () -> bigquery.query(config));
34243544
assertTrue(
34253545
e.getMessage()
3426-
.contains("Arrow results format for slow query path execution is not yet supported."));
3546+
.contains(
3547+
"Query is incomplete, but no job reference was returned to await completion."));
3548+
}
3549+
3550+
@Test
3551+
void testQueryWithArrowFormatIncompleteJobJobNotFound() throws Exception {
3552+
JobId queryJob = JobId.of(PROJECT, JOB).toBuilder().setLocation(LOCATION).build();
3553+
com.google.api.services.bigquery.model.QueryResponse queryResponsePb =
3554+
new com.google.api.services.bigquery.model.QueryResponse()
3555+
.setQueryId("q-arrow-incomplete")
3556+
.setJobComplete(false)
3557+
.setJobReference(queryJob.toPb());
3558+
3559+
when(bigqueryRpcMock.queryRpcSkipExceptionTranslation(eq(PROJECT), any(QueryRequest.class)))
3560+
.thenReturn(queryResponsePb);
3561+
when(bigqueryRpcMock.getJobSkipExceptionTranslation(eq(PROJECT), eq(JOB), any(), any()))
3562+
.thenThrow(new BigQueryException(404, "Not Found"));
3563+
3564+
bigquery = options.getService();
3565+
3566+
QueryJobConfiguration config =
3567+
QueryJobConfiguration.newBuilder("SELECT id FROM test")
3568+
.setQueryResultsFormat(QueryResultsFormat.ARROW)
3569+
.build();
3570+
3571+
BigQueryException e = assertThrows(BigQueryException.class, () -> bigquery.query(config));
3572+
assertTrue(e.getMessage().contains("Job no longer exists or could not be retrieved"));
34273573
}
34283574

34293575
@Test

0 commit comments

Comments
 (0)