@@ -302,10 +302,12 @@ static class ArrowQueryPageFetcher implements NextPageFetcher<FieldValueList> {
302302 private static final long DEFAULT_PAGE_SIZE = 10000L ;
303303
304304 private final JobId jobId ;
305+ private final String customStreamName ;
305306 private final Schema schema ;
306307 private final byte [] arrowSchemaBytes ;
307308 private final BigQueryOptions serviceOptions ;
308309 private final long maxResults ;
310+ private final long pageSize ;
309311 private final Map <BigQueryRpc .Option , ?> optionsMap ;
310312
311313 private transient org .apache .arrow .vector .types .pojo .Schema arrowSchemaPojo ;
@@ -325,42 +327,64 @@ static class ArrowQueryPageFetcher implements NextPageFetcher<FieldValueList> {
325327 long initialRowOffset ,
326328 Long maxResults ,
327329 Map <BigQueryRpc .Option , ?> optionsMap ) {
330+ this (
331+ jobId ,
332+ null ,
333+ schema ,
334+ arrowSchemaBytes ,
335+ arrowSchemaPojo ,
336+ serviceOptions ,
337+ initialRowOffset ,
338+ maxResults ,
339+ optionsMap );
340+ }
341+
342+ ArrowQueryPageFetcher (
343+ JobId jobId ,
344+ String customStreamName ,
345+ Schema schema ,
346+ byte [] arrowSchemaBytes ,
347+ org .apache .arrow .vector .types .pojo .Schema arrowSchemaPojo ,
348+ BigQueryOptions serviceOptions ,
349+ long initialRowOffset ,
350+ Long maxResults ,
351+ Map <BigQueryRpc .Option , ?> optionsMap ) {
328352 this .jobId = jobId ;
353+ this .customStreamName = customStreamName ;
329354 this .schema = schema ;
330355 this .arrowSchemaBytes = arrowSchemaBytes ;
331356 this .arrowSchemaPojo = arrowSchemaPojo ;
332357 this .serviceOptions = serviceOptions ;
333358 this .totalRowsReturned = initialRowOffset ;
334359 this .maxResults = maxResults != null ? maxResults : Long .MAX_VALUE ;
335360 this .optionsMap = optionsMap ;
361+ Number optionPageSize =
362+ optionsMap != null ? (Number ) optionsMap .get (BigQueryRpc .Option .MAX_RESULTS ) : null ;
363+ this .pageSize =
364+ optionPageSize != null && optionPageSize .longValue () > 0
365+ ? optionPageSize .longValue ()
366+ : DEFAULT_PAGE_SIZE ;
336367 }
337368
338369 @ Override
339370 public Page <FieldValueList > getNextPage () {
371+ // Re-initialize transient buffer if the page fetcher was deserialized.
340372 if (buffer == null ) {
341373 buffer = new ArrayDeque <>();
342374 }
343375 if (streamClosed || totalRowsReturned >= maxResults ) {
344- closeClient ();
345376 return null ;
346377 }
347378
348- Number optionPageSize =
349- optionsMap != null ? (Number ) optionsMap .get (BigQueryRpc .Option .MAX_RESULTS ) : null ;
350- long pageSize =
351- optionPageSize != null && optionPageSize .longValue () > 0
352- ? optionPageSize .longValue ()
353- : DEFAULT_PAGE_SIZE ;
354379 List <FieldValueList > rowBatch = new ArrayList <>((int ) Math .min (pageSize , 10000L ));
355380
356381 try {
357382 // Resolve job location in order: JobId location -> BigQueryOptions location -> "global"
358383 // default.
359384 // The Storage Read API stream resource name requires a location component (e.g.
360385 // projects/{project}/locations/{location}/jobs/{job}/streams/_default). If no specific
361- // location
362- // was provided on the job or service options, defaulting to "global" allows queries created
363- // without an explicit location to still stream results without failing.
386+ // location was provided on the job or service options, defaulting to "global" allows
387+ // queries created without an explicit location to still stream results without failing.
364388 String location = jobId .getLocation ();
365389 if (location == null ) {
366390 location = serviceOptions .getLocation ();
@@ -382,13 +406,18 @@ public Page<FieldValueList> getNextPage() {
382406 bqReadClient = ((BigQueryImpl ) service ).getBigQueryReadClient (location );
383407 }
384408
385- // Construct the default stream path for reading job query results via Storage Read API.
386- String streamName =
387- String .format (
388- "projects/%s/locations/%s/jobs/%s/streams/_default" ,
389- jobId .getProject () != null ? jobId .getProject () : serviceOptions .getProjectId (),
390- location ,
391- jobId .getJob ());
409+ String streamName ;
410+ if (customStreamName != null ) {
411+ streamName = customStreamName ;
412+ } else {
413+ // Construct the default stream path for reading job query results via Storage Read API.
414+ streamName =
415+ String .format (
416+ "projects/%s/locations/%s/jobs/%s/streams/_default" ,
417+ jobId .getProject () != null ? jobId .getProject () : serviceOptions .getProjectId (),
418+ location ,
419+ jobId .getJob ());
420+ }
392421
393422 ReadRowsRequest readRowsRequest =
394423 ReadRowsRequest .newBuilder ()
@@ -2530,10 +2559,23 @@ && getOptions().getOpenTelemetryTracer() != null) {
25302559 throw new BigQueryException (bigQueryErrors );
25312560 }
25322561
2533- // If query is incomplete, Arrow format for slow query path is not yet supported.
2562+ // If the query is incomplete (took longer than the fast-path timeout), transition
2563+ // to the slow path: retrieve the created job, wait for completion, and read the
2564+ // results using the Storage Read API in Arrow format.
25342565 if (!Boolean .TRUE .equals (results .getJobComplete ())) {
2535- throw new UnsupportedOperationException (
2536- "Arrow results format for slow query path execution is not yet supported." );
2566+ if (results .getJobReference () == null ) {
2567+ throw new BigQueryException (
2568+ 0 , "Query is incomplete, but no job reference was returned to await completion." );
2569+ }
2570+ JobId jobId = JobId .fromPb (results .getJobReference ());
2571+ Job job = getJob (jobId , options );
2572+ if (job == null ) {
2573+ throw new BigQueryException (0 , "Job no longer exists or could not be retrieved: " + jobId );
2574+ }
2575+ Job completedJob = job .waitFor ();
2576+ Long maxResults =
2577+ content .getMaxResults () != null ? content .getMaxResults ().longValue () : null ;
2578+ return readArrowTableResultFromJob (completedJob , maxResults , null , options );
25372579 }
25382580
25392581 // If the query completed but the Arrow schema is missing from the response, fail fast.
@@ -2710,8 +2752,7 @@ && getOptions().getOpenTelemetryTracer() != null) {
27102752 return queryRpc (projectId , content , options );
27112753 }
27122754 if (configuration .getQueryResultsFormat () == QueryResultsFormat .ARROW ) {
2713- throw new UnsupportedOperationException (
2714- "Arrow results format for slow query path execution is not yet supported." );
2755+ return queryFallbackArrow (jobId , configuration , options );
27152756 }
27162757 return create (JobInfo .of (jobId , configuration ), options );
27172758 } finally {
@@ -2990,6 +3031,205 @@ private ArrowQueryResult createArrowQueryResultFromTable(
29903031 return ArrowQueryResultImpl .fromReadSession (readSession , jobId , client );
29913032 }
29923033
3034+ /**
3035+ * Reads query result rows from a completed query job's destination table using the Storage Read
3036+ * API in Arrow format, returning a populated {@link TableResult}.
3037+ *
3038+ * @param completedJob the completed query job
3039+ * @param maxResults maximum results requested, or {@code null}
3040+ * @param fallbackDestinationTable fallback destination table if not present on the job
3041+ * @param options query job options
3042+ * @return a {@link TableResult} containing decoded rows and execution metadata
3043+ * @throws BigQueryException if job failed or ReadSession creation fails
3044+ */
3045+ private TableResult readArrowTableResultFromJob (
3046+ Job completedJob , Long maxResults , TableId fallbackDestinationTable , JobOption ... options ) {
3047+ if (completedJob == null ) {
3048+ throw new BigQueryException (0 , "Job no longer exists or could not be retrieved." );
3049+ }
3050+
3051+ if (completedJob .getStatus ().getError () != null ) {
3052+ throw new BigQueryException (Collections .singletonList (completedJob .getStatus ().getError ()));
3053+ }
3054+
3055+ // Resolve the destination table containing the query results.
3056+ TableId destinationTable = null ;
3057+ if (completedJob .getConfiguration () instanceof QueryJobConfiguration ) {
3058+ destinationTable =
3059+ ((QueryJobConfiguration ) completedJob .getConfiguration ()).getDestinationTable ();
3060+ }
3061+ if (destinationTable == null ) {
3062+ destinationTable = fallbackDestinationTable ;
3063+ }
3064+ if (destinationTable == null ) {
3065+ throw new BigQueryException (0 , "Unable to resolve destination table for query job" );
3066+ }
3067+
3068+ // Extract query execution statistics from the completed job metadata.
3069+ JobStatistics .QueryStatistics stats =
3070+ completedJob .getStatistics () instanceof JobStatistics .QueryStatistics
3071+ ? (JobStatistics .QueryStatistics ) completedJob .getStatistics ()
3072+ : null ;
3073+
3074+ StatementType statementType = stats != null ? stats .getStatementType () : null ;
3075+ Long totalBytesBilled = stats != null ? stats .getTotalBytesBilled () : null ;
3076+ Long totalBytesProcessed = stats != null ? stats .getTotalBytesProcessed () : null ;
3077+ Long totalSlotMs = stats != null ? stats .getTotalSlotMs () : null ;
3078+ Long numDmlAffectedRows = stats != null ? stats .getNumDmlAffectedRows () : null ;
3079+ SessionInfo sessionInfo = stats != null ? stats .getSessionInfo () : null ;
3080+
3081+ // If the completed job is a DDL or DML statement, return immediately without
3082+ // making a Storage Read API createReadSession network call.
3083+ boolean isDml = numDmlAffectedRows != null ;
3084+ boolean isDdl = stats != null && stats .getDdlOperationPerformed () != null ;
3085+ if (isDml || isDdl ) {
3086+ return TableResult .newBuilder ()
3087+ .setSchema (stats != null ? stats .getSchema () : null )
3088+ .setTotalRows (isDml ? numDmlAffectedRows : 0L )
3089+ .setPageNoSchema (new PageImpl <>(null , null , ImmutableList .of ()))
3090+ .setJobId (completedJob .getJobId ())
3091+ .setRowsInPage (0L )
3092+ .setStatementType (statementType )
3093+ .setTotalBytesBilled (totalBytesBilled )
3094+ .setTotalBytesProcessed (totalBytesProcessed )
3095+ .setTotalSlotMs (totalSlotMs )
3096+ .setNumDmlAffectedRows (numDmlAffectedRows )
3097+ .setSessionInfo (sessionInfo )
3098+ .build ();
3099+ }
3100+
3101+ // Create a Storage Read API ReadSession targeting the destination table in Arrow format.
3102+ String destProject = destinationTable .getProject ();
3103+ if (destProject == null && completedJob .getJobId () != null ) {
3104+ destProject = completedJob .getJobId ().getProject ();
3105+ }
3106+ if (destProject == null ) {
3107+ destProject = getOptions ().getProjectId ();
3108+ }
3109+ String parent = String .format ("projects/%s" , destProject );
3110+ String srcTable =
3111+ String .format (
3112+ "projects/%s/datasets/%s/tables/%s" ,
3113+ destProject , destinationTable .getDataset (), destinationTable .getTable ());
3114+
3115+ BigQueryReadClient client = getBigQueryReadClient ();
3116+ CreateReadSessionRequest request =
3117+ CreateReadSessionRequest .newBuilder ()
3118+ .setParent (parent )
3119+ .setReadSession (
3120+ ReadSession .newBuilder ().setTable (srcTable ).setDataFormat (DataFormat .ARROW ))
3121+ .setMaxStreamCount (1 )
3122+ .build ();
3123+ ReadSession readSession ;
3124+ try {
3125+ readSession = client .createReadSession (request );
3126+ } catch (Exception e ) {
3127+ throw new BigQueryException (0 , "Failed to create ReadSession for query job" , e );
3128+ }
3129+
3130+ // Deserialize the Arrow schema and convert to BigQuery Schema for TableResult metadata.
3131+ org .apache .arrow .vector .types .pojo .Schema arrowSchemaPojo = null ;
3132+ byte [] arrowSchemaBytes = null ;
3133+ if (readSession .hasArrowSchema ()) {
3134+ arrowSchemaBytes = readSession .getArrowSchema ().getSerializedSchema ().toByteArray ();
3135+ try {
3136+ arrowSchemaPojo = ArrowDeserializer .deserializeSchema (arrowSchemaBytes );
3137+ } catch (IOException e ) {
3138+ throw new BigQueryException (0 , "Failed to deserialize Arrow schema from ReadSession" , e );
3139+ }
3140+ }
3141+ Schema schema =
3142+ arrowSchemaPojo != null
3143+ ? ArrowPojoUtils .arrowSchemaToBigQuerySchema (arrowSchemaPojo )
3144+ : (stats != null ? stats .getSchema () : null );
3145+
3146+ String streamName =
3147+ readSession .getStreamsCount () > 0 ? readSession .getStreams (0 ).getName () : null ;
3148+
3149+ // If the destination table has no data streams (e.g. empty SELECT query results),
3150+ // return an empty TableResult populated with execution statistics.
3151+ if (streamName == null ) {
3152+ return TableResult .newBuilder ()
3153+ .setSchema (schema )
3154+ .setTotalRows (numDmlAffectedRows != null ? numDmlAffectedRows : 0L )
3155+ .setPageNoSchema (new PageImpl <>(null , null , ImmutableList .of ()))
3156+ .setJobId (completedJob .getJobId ())
3157+ .setRowsInPage (0L )
3158+ .setStatementType (statementType )
3159+ .setTotalBytesBilled (totalBytesBilled )
3160+ .setTotalBytesProcessed (totalBytesProcessed )
3161+ .setTotalSlotMs (totalSlotMs )
3162+ .setNumDmlAffectedRows (numDmlAffectedRows )
3163+ .setSessionInfo (sessionInfo )
3164+ .build ();
3165+ }
3166+
3167+ // Initialize the page fetcher targeting the ReadSession stream to load the first page of rows.
3168+ ArrowQueryPageFetcher pageFetcher =
3169+ new ArrowQueryPageFetcher (
3170+ completedJob .getJobId (),
3171+ streamName ,
3172+ schema ,
3173+ arrowSchemaBytes ,
3174+ arrowSchemaPojo ,
3175+ getOptions (),
3176+ 0L ,
3177+ maxResults ,
3178+ optionMap (options ));
3179+
3180+ Page <FieldValueList > firstPage = pageFetcher .getNextPage ();
3181+ List <FieldValueList > firstPageRows =
3182+ firstPage != null ? ImmutableList .copyOf (firstPage .getValues ()) : ImmutableList .of ();
3183+ long rowsInPage = (long ) firstPageRows .size ();
3184+
3185+ // Determine total row count: prefer DML affected rows, fallback to estimated row count,
3186+ // or actual first page rows returned.
3187+ long totalRows =
3188+ numDmlAffectedRows != null
3189+ ? numDmlAffectedRows
3190+ : (readSession .getEstimatedRowCount () > 0
3191+ ? readSession .getEstimatedRowCount ()
3192+ : rowsInPage );
3193+
3194+ // Assemble and return the complete TableResult.
3195+ return TableResult .newBuilder ()
3196+ .setSchema (schema )
3197+ .setTotalRows (totalRows )
3198+ .setPageNoSchema (
3199+ firstPage != null ? firstPage : new PageImpl <>(null , null , ImmutableList .of ()))
3200+ .setJobId (completedJob .getJobId ())
3201+ .setRowsInPage (rowsInPage )
3202+ .setStatementType (statementType )
3203+ .setTotalBytesBilled (totalBytesBilled )
3204+ .setTotalBytesProcessed (totalBytesProcessed )
3205+ .setTotalSlotMs (totalSlotMs )
3206+ .setNumDmlAffectedRows (numDmlAffectedRows )
3207+ .setSessionInfo (sessionInfo )
3208+ .build ();
3209+ }
3210+
3211+ /**
3212+ * Executes a slow-path query job using {@code jobs.insert}, awaits its completion, and streams
3213+ * the result rows via the BigQuery Storage Read API in Arrow format, wrapping the decoded rows in
3214+ * a {@link TableResult}.
3215+ *
3216+ * @param jobId the job ID, or {@code null}
3217+ * @param configuration the query job configuration
3218+ * @param options query job options
3219+ * @return a {@link TableResult} containing the decoded rows and job execution metadata
3220+ * @throws InterruptedException if interrupted while awaiting job completion
3221+ * @throws BigQueryException if job execution or ReadSession creation fails
3222+ */
3223+ private TableResult queryFallbackArrow (
3224+ JobId jobId , QueryJobConfiguration configuration , JobOption ... options )
3225+ throws InterruptedException {
3226+ // Submit the query job via jobs.insert and poll until completion.
3227+ Job job = create (JobInfo .of (jobId , configuration ), options );
3228+ Job completedJob = job .waitFor ();
3229+ return readArrowTableResultFromJob (
3230+ completedJob , configuration .getMaxResults (), configuration .getDestinationTable (), options );
3231+ }
3232+
29933233 @ Override
29943234 public QueryResponse getQueryResults (JobId jobId , QueryResultsOption ... options ) {
29953235 Map <BigQueryRpc .Option , ?> optionsMap = optionMap (options );
0 commit comments