Skip to content

Commit 26e568a

Browse files
authored
feat(bigquery): add Storage Read API slow-path fallback for row-based query() (#14409)
This PR implements slow-path execution fallback for row-based queries requesting Arrow results format (`QueryResultsFormat.ARROW`). When queries cannot be evaluated via the fast query path (such as queries writing to destination tables or exceeding fast-path limits), BigQuery job execution is triggered and table results are streamed via the BigQuery Storage Read API in Arrow format. Follow-up PR stacked on top of #14405.
1 parent 8d12a8f commit 26e568a

2 files changed

Lines changed: 531 additions & 35 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 262 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -302,10 +302,12 @@ static class ArrowQueryPageFetcher implements NextPageFetcher<FieldValueList> {
302302
private static final long DEFAULT_PAGE_SIZE = 10000L;
303303

304304
private final JobId jobId;
305+
private final String customStreamName;
305306
private final Schema schema;
306307
private final byte[] arrowSchemaBytes;
307308
private final BigQueryOptions serviceOptions;
308309
private final long maxResults;
310+
private final long pageSize;
309311
private final Map<BigQueryRpc.Option, ?> optionsMap;
310312

311313
private transient org.apache.arrow.vector.types.pojo.Schema arrowSchemaPojo;
@@ -325,42 +327,64 @@ static class ArrowQueryPageFetcher implements NextPageFetcher<FieldValueList> {
325327
long initialRowOffset,
326328
Long maxResults,
327329
Map<BigQueryRpc.Option, ?> optionsMap) {
330+
this(
331+
jobId,
332+
null,
333+
schema,
334+
arrowSchemaBytes,
335+
arrowSchemaPojo,
336+
serviceOptions,
337+
initialRowOffset,
338+
maxResults,
339+
optionsMap);
340+
}
341+
342+
ArrowQueryPageFetcher(
343+
JobId jobId,
344+
String customStreamName,
345+
Schema schema,
346+
byte[] arrowSchemaBytes,
347+
org.apache.arrow.vector.types.pojo.Schema arrowSchemaPojo,
348+
BigQueryOptions serviceOptions,
349+
long initialRowOffset,
350+
Long maxResults,
351+
Map<BigQueryRpc.Option, ?> optionsMap) {
328352
this.jobId = jobId;
353+
this.customStreamName = customStreamName;
329354
this.schema = schema;
330355
this.arrowSchemaBytes = arrowSchemaBytes;
331356
this.arrowSchemaPojo = arrowSchemaPojo;
332357
this.serviceOptions = serviceOptions;
333358
this.totalRowsReturned = initialRowOffset;
334359
this.maxResults = maxResults != null ? maxResults : Long.MAX_VALUE;
335360
this.optionsMap = optionsMap;
361+
Number optionPageSize =
362+
optionsMap != null ? (Number) optionsMap.get(BigQueryRpc.Option.MAX_RESULTS) : null;
363+
this.pageSize =
364+
optionPageSize != null && optionPageSize.longValue() > 0
365+
? optionPageSize.longValue()
366+
: DEFAULT_PAGE_SIZE;
336367
}
337368

338369
@Override
339370
public Page<FieldValueList> getNextPage() {
371+
// Re-initialize transient buffer if the page fetcher was deserialized.
340372
if (buffer == null) {
341373
buffer = new ArrayDeque<>();
342374
}
343375
if (streamClosed || totalRowsReturned >= maxResults) {
344-
closeClient();
345376
return null;
346377
}
347378

348-
Number optionPageSize =
349-
optionsMap != null ? (Number) optionsMap.get(BigQueryRpc.Option.MAX_RESULTS) : null;
350-
long pageSize =
351-
optionPageSize != null && optionPageSize.longValue() > 0
352-
? optionPageSize.longValue()
353-
: DEFAULT_PAGE_SIZE;
354379
List<FieldValueList> rowBatch = new ArrayList<>((int) Math.min(pageSize, 10000L));
355380

356381
try {
357382
// Resolve job location in order: JobId location -> BigQueryOptions location -> "global"
358383
// default.
359384
// The Storage Read API stream resource name requires a location component (e.g.
360385
// projects/{project}/locations/{location}/jobs/{job}/streams/_default). If no specific
361-
// location
362-
// was provided on the job or service options, defaulting to "global" allows queries created
363-
// without an explicit location to still stream results without failing.
386+
// location was provided on the job or service options, defaulting to "global" allows
387+
// queries created without an explicit location to still stream results without failing.
364388
String location = jobId.getLocation();
365389
if (location == null) {
366390
location = serviceOptions.getLocation();
@@ -382,13 +406,18 @@ public Page<FieldValueList> getNextPage() {
382406
bqReadClient = ((BigQueryImpl) service).getBigQueryReadClient(location);
383407
}
384408

385-
// Construct the default stream path for reading job query results via Storage Read API.
386-
String streamName =
387-
String.format(
388-
"projects/%s/locations/%s/jobs/%s/streams/_default",
389-
jobId.getProject() != null ? jobId.getProject() : serviceOptions.getProjectId(),
390-
location,
391-
jobId.getJob());
409+
String streamName;
410+
if (customStreamName != null) {
411+
streamName = customStreamName;
412+
} else {
413+
// Construct the default stream path for reading job query results via Storage Read API.
414+
streamName =
415+
String.format(
416+
"projects/%s/locations/%s/jobs/%s/streams/_default",
417+
jobId.getProject() != null ? jobId.getProject() : serviceOptions.getProjectId(),
418+
location,
419+
jobId.getJob());
420+
}
392421

393422
ReadRowsRequest readRowsRequest =
394423
ReadRowsRequest.newBuilder()
@@ -2530,10 +2559,23 @@ && getOptions().getOpenTelemetryTracer() != null) {
25302559
throw new BigQueryException(bigQueryErrors);
25312560
}
25322561

2533-
// If query is incomplete, Arrow format for slow query path is not yet supported.
2562+
// If the query is incomplete (took longer than the fast-path timeout), transition
2563+
// to the slow path: retrieve the created job, wait for completion, and read the
2564+
// results using the Storage Read API in Arrow format.
25342565
if (!Boolean.TRUE.equals(results.getJobComplete())) {
2535-
throw new UnsupportedOperationException(
2536-
"Arrow results format for slow query path execution is not yet supported.");
2566+
if (results.getJobReference() == null) {
2567+
throw new BigQueryException(
2568+
0, "Query is incomplete, but no job reference was returned to await completion.");
2569+
}
2570+
JobId jobId = JobId.fromPb(results.getJobReference());
2571+
Job job = getJob(jobId, options);
2572+
if (job == null) {
2573+
throw new BigQueryException(0, "Job no longer exists or could not be retrieved: " + jobId);
2574+
}
2575+
Job completedJob = job.waitFor();
2576+
Long maxResults =
2577+
content.getMaxResults() != null ? content.getMaxResults().longValue() : null;
2578+
return readArrowTableResultFromJob(completedJob, maxResults, null, options);
25372579
}
25382580

25392581
// If the query completed but the Arrow schema is missing from the response, fail fast.
@@ -2710,8 +2752,7 @@ && getOptions().getOpenTelemetryTracer() != null) {
27102752
return queryRpc(projectId, content, options);
27112753
}
27122754
if (configuration.getQueryResultsFormat() == QueryResultsFormat.ARROW) {
2713-
throw new UnsupportedOperationException(
2714-
"Arrow results format for slow query path execution is not yet supported.");
2755+
return queryFallbackArrow(jobId, configuration, options);
27152756
}
27162757
return create(JobInfo.of(jobId, configuration), options);
27172758
} finally {
@@ -2990,6 +3031,205 @@ private ArrowQueryResult createArrowQueryResultFromTable(
29903031
return ArrowQueryResultImpl.fromReadSession(readSession, jobId, client);
29913032
}
29923033

3034+
/**
3035+
* Reads query result rows from a completed query job's destination table using the Storage Read
3036+
* API in Arrow format, returning a populated {@link TableResult}.
3037+
*
3038+
* @param completedJob the completed query job
3039+
* @param maxResults maximum results requested, or {@code null}
3040+
* @param fallbackDestinationTable fallback destination table if not present on the job
3041+
* @param options query job options
3042+
* @return a {@link TableResult} containing decoded rows and execution metadata
3043+
* @throws BigQueryException if job failed or ReadSession creation fails
3044+
*/
3045+
private TableResult readArrowTableResultFromJob(
3046+
Job completedJob, Long maxResults, TableId fallbackDestinationTable, JobOption... options) {
3047+
if (completedJob == null) {
3048+
throw new BigQueryException(0, "Job no longer exists or could not be retrieved.");
3049+
}
3050+
3051+
if (completedJob.getStatus().getError() != null) {
3052+
throw new BigQueryException(Collections.singletonList(completedJob.getStatus().getError()));
3053+
}
3054+
3055+
// Resolve the destination table containing the query results.
3056+
TableId destinationTable = null;
3057+
if (completedJob.getConfiguration() instanceof QueryJobConfiguration) {
3058+
destinationTable =
3059+
((QueryJobConfiguration) completedJob.getConfiguration()).getDestinationTable();
3060+
}
3061+
if (destinationTable == null) {
3062+
destinationTable = fallbackDestinationTable;
3063+
}
3064+
if (destinationTable == null) {
3065+
throw new BigQueryException(0, "Unable to resolve destination table for query job");
3066+
}
3067+
3068+
// Extract query execution statistics from the completed job metadata.
3069+
JobStatistics.QueryStatistics stats =
3070+
completedJob.getStatistics() instanceof JobStatistics.QueryStatistics
3071+
? (JobStatistics.QueryStatistics) completedJob.getStatistics()
3072+
: null;
3073+
3074+
StatementType statementType = stats != null ? stats.getStatementType() : null;
3075+
Long totalBytesBilled = stats != null ? stats.getTotalBytesBilled() : null;
3076+
Long totalBytesProcessed = stats != null ? stats.getTotalBytesProcessed() : null;
3077+
Long totalSlotMs = stats != null ? stats.getTotalSlotMs() : null;
3078+
Long numDmlAffectedRows = stats != null ? stats.getNumDmlAffectedRows() : null;
3079+
SessionInfo sessionInfo = stats != null ? stats.getSessionInfo() : null;
3080+
3081+
// If the completed job is a DDL or DML statement, return immediately without
3082+
// making a Storage Read API createReadSession network call.
3083+
boolean isDml = numDmlAffectedRows != null;
3084+
boolean isDdl = stats != null && stats.getDdlOperationPerformed() != null;
3085+
if (isDml || isDdl) {
3086+
return TableResult.newBuilder()
3087+
.setSchema(stats != null ? stats.getSchema() : null)
3088+
.setTotalRows(isDml ? numDmlAffectedRows : 0L)
3089+
.setPageNoSchema(new PageImpl<>(null, null, ImmutableList.of()))
3090+
.setJobId(completedJob.getJobId())
3091+
.setRowsInPage(0L)
3092+
.setStatementType(statementType)
3093+
.setTotalBytesBilled(totalBytesBilled)
3094+
.setTotalBytesProcessed(totalBytesProcessed)
3095+
.setTotalSlotMs(totalSlotMs)
3096+
.setNumDmlAffectedRows(numDmlAffectedRows)
3097+
.setSessionInfo(sessionInfo)
3098+
.build();
3099+
}
3100+
3101+
// Create a Storage Read API ReadSession targeting the destination table in Arrow format.
3102+
String destProject = destinationTable.getProject();
3103+
if (destProject == null && completedJob.getJobId() != null) {
3104+
destProject = completedJob.getJobId().getProject();
3105+
}
3106+
if (destProject == null) {
3107+
destProject = getOptions().getProjectId();
3108+
}
3109+
String parent = String.format("projects/%s", destProject);
3110+
String srcTable =
3111+
String.format(
3112+
"projects/%s/datasets/%s/tables/%s",
3113+
destProject, destinationTable.getDataset(), destinationTable.getTable());
3114+
3115+
BigQueryReadClient client = getBigQueryReadClient();
3116+
CreateReadSessionRequest request =
3117+
CreateReadSessionRequest.newBuilder()
3118+
.setParent(parent)
3119+
.setReadSession(
3120+
ReadSession.newBuilder().setTable(srcTable).setDataFormat(DataFormat.ARROW))
3121+
.setMaxStreamCount(1)
3122+
.build();
3123+
ReadSession readSession;
3124+
try {
3125+
readSession = client.createReadSession(request);
3126+
} catch (Exception e) {
3127+
throw new BigQueryException(0, "Failed to create ReadSession for query job", e);
3128+
}
3129+
3130+
// Deserialize the Arrow schema and convert to BigQuery Schema for TableResult metadata.
3131+
org.apache.arrow.vector.types.pojo.Schema arrowSchemaPojo = null;
3132+
byte[] arrowSchemaBytes = null;
3133+
if (readSession.hasArrowSchema()) {
3134+
arrowSchemaBytes = readSession.getArrowSchema().getSerializedSchema().toByteArray();
3135+
try {
3136+
arrowSchemaPojo = ArrowDeserializer.deserializeSchema(arrowSchemaBytes);
3137+
} catch (IOException e) {
3138+
throw new BigQueryException(0, "Failed to deserialize Arrow schema from ReadSession", e);
3139+
}
3140+
}
3141+
Schema schema =
3142+
arrowSchemaPojo != null
3143+
? ArrowPojoUtils.arrowSchemaToBigQuerySchema(arrowSchemaPojo)
3144+
: (stats != null ? stats.getSchema() : null);
3145+
3146+
String streamName =
3147+
readSession.getStreamsCount() > 0 ? readSession.getStreams(0).getName() : null;
3148+
3149+
// If the destination table has no data streams (e.g. empty SELECT query results),
3150+
// return an empty TableResult populated with execution statistics.
3151+
if (streamName == null) {
3152+
return TableResult.newBuilder()
3153+
.setSchema(schema)
3154+
.setTotalRows(numDmlAffectedRows != null ? numDmlAffectedRows : 0L)
3155+
.setPageNoSchema(new PageImpl<>(null, null, ImmutableList.of()))
3156+
.setJobId(completedJob.getJobId())
3157+
.setRowsInPage(0L)
3158+
.setStatementType(statementType)
3159+
.setTotalBytesBilled(totalBytesBilled)
3160+
.setTotalBytesProcessed(totalBytesProcessed)
3161+
.setTotalSlotMs(totalSlotMs)
3162+
.setNumDmlAffectedRows(numDmlAffectedRows)
3163+
.setSessionInfo(sessionInfo)
3164+
.build();
3165+
}
3166+
3167+
// Initialize the page fetcher targeting the ReadSession stream to load the first page of rows.
3168+
ArrowQueryPageFetcher pageFetcher =
3169+
new ArrowQueryPageFetcher(
3170+
completedJob.getJobId(),
3171+
streamName,
3172+
schema,
3173+
arrowSchemaBytes,
3174+
arrowSchemaPojo,
3175+
getOptions(),
3176+
0L,
3177+
maxResults,
3178+
optionMap(options));
3179+
3180+
Page<FieldValueList> firstPage = pageFetcher.getNextPage();
3181+
List<FieldValueList> firstPageRows =
3182+
firstPage != null ? ImmutableList.copyOf(firstPage.getValues()) : ImmutableList.of();
3183+
long rowsInPage = (long) firstPageRows.size();
3184+
3185+
// Determine total row count: prefer DML affected rows, fallback to estimated row count,
3186+
// or actual first page rows returned.
3187+
long totalRows =
3188+
numDmlAffectedRows != null
3189+
? numDmlAffectedRows
3190+
: (readSession.getEstimatedRowCount() > 0
3191+
? readSession.getEstimatedRowCount()
3192+
: rowsInPage);
3193+
3194+
// Assemble and return the complete TableResult.
3195+
return TableResult.newBuilder()
3196+
.setSchema(schema)
3197+
.setTotalRows(totalRows)
3198+
.setPageNoSchema(
3199+
firstPage != null ? firstPage : new PageImpl<>(null, null, ImmutableList.of()))
3200+
.setJobId(completedJob.getJobId())
3201+
.setRowsInPage(rowsInPage)
3202+
.setStatementType(statementType)
3203+
.setTotalBytesBilled(totalBytesBilled)
3204+
.setTotalBytesProcessed(totalBytesProcessed)
3205+
.setTotalSlotMs(totalSlotMs)
3206+
.setNumDmlAffectedRows(numDmlAffectedRows)
3207+
.setSessionInfo(sessionInfo)
3208+
.build();
3209+
}
3210+
3211+
/**
3212+
* Executes a slow-path query job using {@code jobs.insert}, awaits its completion, and streams
3213+
* the result rows via the BigQuery Storage Read API in Arrow format, wrapping the decoded rows in
3214+
* a {@link TableResult}.
3215+
*
3216+
* @param jobId the job ID, or {@code null}
3217+
* @param configuration the query job configuration
3218+
* @param options query job options
3219+
* @return a {@link TableResult} containing the decoded rows and job execution metadata
3220+
* @throws InterruptedException if interrupted while awaiting job completion
3221+
* @throws BigQueryException if job execution or ReadSession creation fails
3222+
*/
3223+
private TableResult queryFallbackArrow(
3224+
JobId jobId, QueryJobConfiguration configuration, JobOption... options)
3225+
throws InterruptedException {
3226+
// Submit the query job via jobs.insert and poll until completion.
3227+
Job job = create(JobInfo.of(jobId, configuration), options);
3228+
Job completedJob = job.waitFor();
3229+
return readArrowTableResultFromJob(
3230+
completedJob, configuration.getMaxResults(), configuration.getDestinationTable(), options);
3231+
}
3232+
29933233
@Override
29943234
public QueryResponse getQueryResults(JobId jobId, QueryResultsOption... options) {
29953235
Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options);

0 commit comments

Comments
 (0)