Skip to content

Commit 8d12a8f

Browse files
authored
feat(bigquery): accelerate row-based query() with Arrow wire format (#14405)
Enables Apache Arrow wire acceleration for the traditional `BigQuery.query()` API returning row-based `TableResult`. Part of the BigQuery Apache Arrow support stack. Based on #14404 (page fetcher).
1 parent 615409f commit 8d12a8f

2 files changed

Lines changed: 602 additions & 72 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 200 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import com.google.common.collect.Iterables;
6868
import com.google.common.collect.Lists;
6969
import com.google.common.collect.Maps;
70+
import com.google.common.primitives.Longs;
7071
import io.opentelemetry.api.common.Attributes;
7172
import io.opentelemetry.api.trace.Span;
7273
import io.opentelemetry.context.Scope;
@@ -2408,23 +2409,11 @@ public com.google.api.services.bigquery.model.QueryResponse call()
24082409
return job;
24092410
}
24102411

2411-
StatementType statementType =
2412-
results.getStatementType() != null
2413-
? StatementType.valueOf(results.getStatementType())
2414-
: null;
2415-
Long totalBytesBilled = results.getTotalBytesBilled();
2416-
Long totalBytesProcessed = results.getTotalBytesProcessed();
2417-
Long totalSlotMs = results.getTotalSlotMs();
2418-
Long numDmlAffectedRows = results.getNumDmlAffectedRows();
2419-
SessionInfo sessionInfo =
2420-
results.getSessionInfo() != null ? SessionInfo.fromPb(results.getSessionInfo()) : null;
2421-
24222412
if (results.getPageToken() != null) {
24232413
JobId jobId = JobId.fromPb(results.getJobReference());
24242414
String cursor = results.getPageToken();
2425-
return TableResult.newBuilder()
2415+
return newTableResultBuilder(results)
24262416
.setSchema(schema)
2427-
.setTotalRows(numRows)
24282417
.setPageNoSchema(
24292418
new PageImpl<>(
24302419
// fetch next pages of results
@@ -2435,21 +2424,12 @@ public com.google.api.services.bigquery.model.QueryResponse call()
24352424
schema,
24362425
getOptions().getDataFormatOptions().useInt64Timestamp())))
24372426
.setJobId(jobId)
2438-
.setQueryId(results.getQueryId())
2439-
.setJobCreationReason(JobCreationReason.fromPb(results.getJobCreationReason()))
24402427
.setRowsInPage(results.getRows() != null ? (long) results.getRows().size() : 0L)
2441-
.setStatementType(statementType)
2442-
.setTotalBytesBilled(totalBytesBilled)
2443-
.setTotalBytesProcessed(totalBytesProcessed)
2444-
.setTotalSlotMs(totalSlotMs)
2445-
.setNumDmlAffectedRows(numDmlAffectedRows)
2446-
.setSessionInfo(sessionInfo)
24472428
.build();
24482429
}
24492430
// only 1 page of result
2450-
return TableResult.newBuilder()
2431+
return newTableResultBuilder(results)
24512432
.setSchema(schema)
2452-
.setTotalRows(numRows)
24532433
.setPageNoSchema(
24542434
new PageImpl<>(
24552435
new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
@@ -2461,15 +2441,198 @@ public com.google.api.services.bigquery.model.QueryResponse call()
24612441
// Return the JobID of the successful job
24622442
.setJobId(
24632443
results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
2464-
.setQueryId(results.getQueryId())
2465-
.setJobCreationReason(JobCreationReason.fromPb(results.getJobCreationReason()))
24662444
.setRowsInPage(results.getRows() != null ? (long) results.getRows().size() : 0L)
2445+
.build();
2446+
}
2447+
2448+
private static TableResult.Builder newTableResultBuilder(
2449+
com.google.api.services.bigquery.model.QueryResponse results) {
2450+
StatementType statementType =
2451+
results.getStatementType() != null
2452+
? StatementType.valueOf(results.getStatementType())
2453+
: null;
2454+
return TableResult.newBuilder()
2455+
.setTotalRows(getNumRows(results))
2456+
.setQueryId(results.getQueryId())
2457+
.setJobCreationReason(
2458+
results.getJobCreationReason() != null
2459+
? JobCreationReason.fromPb(results.getJobCreationReason())
2460+
: null)
24672461
.setStatementType(statementType)
2468-
.setTotalBytesBilled(totalBytesBilled)
2469-
.setTotalBytesProcessed(totalBytesProcessed)
2470-
.setTotalSlotMs(totalSlotMs)
2471-
.setNumDmlAffectedRows(numDmlAffectedRows)
2472-
.setSessionInfo(sessionInfo)
2462+
.setTotalBytesBilled(results.getTotalBytesBilled())
2463+
.setTotalBytesProcessed(results.getTotalBytesProcessed())
2464+
.setTotalSlotMs(results.getTotalSlotMs())
2465+
.setNumDmlAffectedRows(results.getNumDmlAffectedRows())
2466+
.setSessionInfo(
2467+
results.getSessionInfo() != null ? SessionInfo.fromPb(results.getSessionInfo()) : null);
2468+
}
2469+
2470+
private static long getNumRows(com.google.api.services.bigquery.model.QueryResponse results) {
2471+
if (results.getNumDmlAffectedRows() == null && results.getTotalRows() == null) {
2472+
return 0L;
2473+
} else if (results.getNumDmlAffectedRows() != null) {
2474+
return results.getNumDmlAffectedRows();
2475+
} else {
2476+
return results.getTotalRows().longValue();
2477+
}
2478+
}
2479+
2480+
/**
2481+
* Executes a fast-path query RPC request expecting Arrow-formatted wire response.
2482+
*
2483+
* <p>Deserializes the returned Arrow IPC schema and record batch into standard {@link
2484+
* TableResult} row-based representations, configuring {@link ArrowQueryPageFetcher} for
2485+
* subsequent pages when pagination tokens are returned.
2486+
*
2487+
* @param projectId project ID in which to execute the query
2488+
* @param content query request content PB
2489+
* @param options job options
2490+
* @return either a {@link Job} if incomplete/fallback is required, or {@link TableResult}
2491+
* @throws InterruptedException if interrupted while awaiting RPC execution
2492+
*/
2493+
private Object queryRpcArrow(
2494+
final String projectId, final QueryRequest content, JobOption... options)
2495+
throws InterruptedException {
2496+
com.google.api.services.bigquery.model.QueryResponse results;
2497+
Span queryRpc = null;
2498+
if (getOptions().isOpenTelemetryTracingEnabled()
2499+
&& getOptions().getOpenTelemetryTracer() != null) {
2500+
queryRpc =
2501+
getOptions()
2502+
.getOpenTelemetryTracer()
2503+
.spanBuilder("com.google.cloud.bigquery.BigQuery.queryRpc")
2504+
.setAttribute("bq.query.project_id", projectId)
2505+
.setAllAttributes(otelAttributesFromQueryRequest(content))
2506+
.setAllAttributes(otelAttributesFromOptions(options))
2507+
.startSpan();
2508+
}
2509+
try (Scope queryRpcScope = queryRpc != null ? queryRpc.makeCurrent() : null) {
2510+
results =
2511+
BigQueryRetryHelper.runWithRetries(
2512+
() -> bigQueryRpc.queryRpcSkipExceptionTranslation(projectId, content),
2513+
getOptions().getRetrySettings(),
2514+
getOptions().getResultRetryAlgorithm(),
2515+
getOptions().getClock(),
2516+
DEFAULT_RETRY_CONFIG,
2517+
getOptions().isOpenTelemetryTracingEnabled(),
2518+
getOptions().getOpenTelemetryTracer());
2519+
} catch (BigQueryRetryHelper.BigQueryRetryHelperException e) {
2520+
throw BigQueryException.translateAndThrow(e);
2521+
} finally {
2522+
if (queryRpc != null) {
2523+
queryRpc.end();
2524+
}
2525+
}
2526+
2527+
if (results.getErrors() != null) {
2528+
List<BigQueryError> bigQueryErrors =
2529+
Lists.transform(results.getErrors(), BigQueryError.FROM_PB_FUNCTION);
2530+
throw new BigQueryException(bigQueryErrors);
2531+
}
2532+
2533+
// If query is incomplete, Arrow format for slow query path is not yet supported.
2534+
if (!Boolean.TRUE.equals(results.getJobComplete())) {
2535+
throw new UnsupportedOperationException(
2536+
"Arrow results format for slow query path execution is not yet supported.");
2537+
}
2538+
2539+
// If the query completed but the Arrow schema is missing from the response, fail fast.
2540+
if (results.getArrowSchema() == null
2541+
|| results.getArrowSchema().getSerializedSchema() == null) {
2542+
throw new BigQueryException(0, "Arrow schema is missing from the response");
2543+
}
2544+
2545+
// Deserialize Arrow IPC schema and convert to veneer BigQuery Schema.
2546+
byte[] arrowSchemaBytes = results.getArrowSchema().decodeSerializedSchema();
2547+
org.apache.arrow.vector.types.pojo.Schema arrowSchemaPojo;
2548+
try {
2549+
arrowSchemaPojo = ArrowDeserializer.deserializeSchema(arrowSchemaBytes);
2550+
} catch (IOException e) {
2551+
throw new BigQueryException(0, "Failed to deserialize Arrow schema from response", e);
2552+
}
2553+
Schema schema = ArrowPojoUtils.arrowSchemaToBigQuerySchema(arrowSchemaPojo);
2554+
2555+
// Deserialize first page of rows from the Arrow record batch (if present).
2556+
List<FieldValueList> firstPageRows;
2557+
if (results.getArrowRecordBatch() == null
2558+
|| results.getArrowRecordBatch().getSerializedRecordBatch() == null) {
2559+
firstPageRows = ImmutableList.of();
2560+
} else {
2561+
try {
2562+
firstPageRows =
2563+
ArrowDeserializer.deserializeRecordBatch(
2564+
results.getArrowRecordBatch().decodeSerializedRecordBatch(),
2565+
schema,
2566+
arrowSchemaPojo);
2567+
} catch (IOException e) {
2568+
throw new BigQueryException(0, "Failed to deserialize Arrow record batch", e);
2569+
}
2570+
}
2571+
2572+
// Enforce maxResults limit on the first page if requested.
2573+
if (content.getMaxResults() != null && firstPageRows.size() > content.getMaxResults()) {
2574+
firstPageRows =
2575+
ImmutableList.copyOf(Iterables.limit(firstPageRows, content.getMaxResults().intValue()));
2576+
}
2577+
2578+
// Calculate row offset and determine if subsequent pages exist.
2579+
boolean hasMorePages = results.getPageToken() != null;
2580+
long initialRowOffset = 0L;
2581+
if (hasMorePages) {
2582+
Long parsedOffset = Longs.tryParse(results.getPageToken());
2583+
if (parsedOffset == null) {
2584+
throw new BigQueryException(
2585+
0,
2586+
String.format(
2587+
"Unable to parse page token '%s' as a numeric row offset for Arrow query pagination",
2588+
results.getPageToken()));
2589+
}
2590+
initialRowOffset = parsedOffset;
2591+
if (content.getMaxResults() != null
2592+
&& (initialRowOffset >= content.getMaxResults()
2593+
|| firstPageRows.size() >= content.getMaxResults())) {
2594+
hasMorePages = false;
2595+
}
2596+
}
2597+
2598+
// Multi-page results: configure ArrowQueryPageFetcher for subsequent tabledata.list calls.
2599+
if (hasMorePages) {
2600+
if (results.getJobReference() == null) {
2601+
throw new BigQueryException(
2602+
0, "More pages exist, but no job reference was returned to fetch them.");
2603+
}
2604+
JobId jobId = JobId.fromPb(results.getJobReference());
2605+
String cursor = results.getPageToken();
2606+
NextPageFetcher<FieldValueList> pageFetcher =
2607+
new ArrowQueryPageFetcher(
2608+
jobId,
2609+
schema,
2610+
arrowSchemaBytes,
2611+
arrowSchemaPojo,
2612+
getOptions(),
2613+
initialRowOffset,
2614+
content.getMaxResults(),
2615+
optionMap(options));
2616+
2617+
return newTableResultBuilder(results)
2618+
.setSchema(schema)
2619+
.setPageNoSchema(new PageImpl<>(pageFetcher, cursor, firstPageRows))
2620+
.setJobId(jobId)
2621+
.setRowsInPage((long) firstPageRows.size())
2622+
.build();
2623+
}
2624+
2625+
// only 1 page of result
2626+
return newTableResultBuilder(results)
2627+
.setSchema(schema)
2628+
.setPageNoSchema(
2629+
new PageImpl<>(
2630+
new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
2631+
null,
2632+
firstPageRows))
2633+
.setJobId(
2634+
results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)
2635+
.setRowsInPage((long) firstPageRows.size())
24732636
.build();
24742637
}
24752638

@@ -2489,11 +2652,6 @@ public Object queryWithTimeout(
24892652
throws InterruptedException, JobException {
24902653
Job.checkNotDryRun(configuration, "query");
24912654

2492-
if (configuration.getQueryResultsFormat() == QueryResultsFormat.ARROW) {
2493-
throw new UnsupportedOperationException(
2494-
"QueryResultsFormat.ARROW is not supported with query(). Use queryArrow() instead.");
2495-
}
2496-
24972655
// If JobCreationMode is not explicitly set, update it with default value;
24982656
if (configuration.getJobCreationMode() == null) {
24992657
configuration =
@@ -2546,8 +2704,15 @@ && getOptions().getOpenTelemetryTracer() != null) {
25462704
content.setTimeoutMs(timeoutMs);
25472705
}
25482706

2707+
if (configuration.getQueryResultsFormat() == QueryResultsFormat.ARROW) {
2708+
return queryRpcArrow(projectId, content, options);
2709+
}
25492710
return queryRpc(projectId, content, options);
25502711
}
2712+
if (configuration.getQueryResultsFormat() == QueryResultsFormat.ARROW) {
2713+
throw new UnsupportedOperationException(
2714+
"Arrow results format for slow query path execution is not yet supported.");
2715+
}
25512716
return create(JobInfo.of(jobId, configuration), options);
25522717
} finally {
25532718
if (querySpan != null) {

0 commit comments

Comments
 (0)