Skip to content

Commit 576a42e

Browse files
authored
fix(bigquery): raise when an Arrow batch arrives before its schema (#9681)
## Context & Problem `createQueryArrowStream` turned a record batch that arrived without a schema into an empty `RecordBatch`, silently dropping its rows. The skip has been on `main` since #9596, so this fixes it on its own, outside the Arrow stack. ## Summary of Changes - `decodeArrowPayload` throws `Got an Arrow record batch before its schema.` when it gets batch bytes without a schema, matching the Python client (googleapis/google-cloud-python#18791). - Responses with neither a schema nor a batch (for example DDL/DML) still yield one 0-column `RecordBatch`. ## Verification - New unit test for an inline `jobs.query` response with `arrowRecordBatch` but no `arrowSchema`; it fails without the fix. - `pnpm test` in `handwritten/bigquery`: 551 passing. - `GIT_DIFF_ARG=origin/main node ./bin/linter.mjs --strict`: no problems.
1 parent 2062403 commit 576a42e

2 files changed

Lines changed: 52 additions & 3 deletions

File tree

‎handwritten/bigquery/src/arrow.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -350,20 +350,29 @@ export function decodeArrowPayload(
350350
serializedSchema?: Uint8Array | string,
351351
serializedRecordBatch?: Uint8Array | string,
352352
): RecordBatch[] {
353-
if (!serializedSchema) {
353+
const hasSchema =
354+
serializedSchema !== undefined && serializedSchema.length > 0;
355+
const hasBatch =
356+
serializedRecordBatch !== undefined && serializedRecordBatch.length > 0;
357+
if (!hasSchema) {
358+
// Without a schema the batch can't be decoded; returning an empty batch
359+
// would silently drop its rows.
360+
if (hasBatch) {
361+
throw new Error('Got an Arrow record batch before its schema.');
362+
}
354363
return [new arrow.RecordBatch({})];
355364
}
356365
// BigQuery returns the Arrow IPC Schema message and RecordBatch message as
357366
// separate fields; the reader accepts them as consecutive chunks of one IPC
358367
// stream, which avoids copying them into a single buffer.
359-
const chunks = serializedRecordBatch
368+
const chunks = hasBatch
360369
? [toUint8Array(serializedSchema), toUint8Array(serializedRecordBatch)]
361370
: [toUint8Array(serializedSchema)];
362371
const reader = arrow.RecordBatchReader.from(chunks);
363372
reader.open();
364373
// With no record batch, the reader yields its own 0-row placeholder batch,
365374
// which leaves nested Struct children empty and List children undefined.
366-
return serializedRecordBatch
375+
return hasBatch
367376
? [...reader]
368377
: [createEmptyRecordBatch(arrow, reader.schema)];
369378
}

‎handwritten/bigquery/test/arrow.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,46 @@ describe('Arrow Support', () => {
449449
assert.strictEqual(batches[0].numRows, 2);
450450
});
451451

452+
it('should fail instead of dropping rows when jobs.query returns a record batch without a schema', async () => {
453+
stubJobsQuery(bq, {
454+
res: {
455+
jobComplete: true,
456+
totalRows: '2',
457+
pageRowCount: '2',
458+
arrowRecordBatch: {
459+
serializedRecordBatch: SAMPLE_PAYLOAD.serializedRecordBatch,
460+
},
461+
},
462+
});
463+
464+
// Verify the batch isn't silently replaced with an empty RecordBatch.
465+
await assert.rejects(
466+
bq.createQueryArrowStream('SELECT 1').toArray(),
467+
/Got an Arrow record batch before its schema\./,
468+
);
469+
});
470+
471+
it('should treat an empty record batch without a schema like a missing one', async () => {
472+
stubJobsQuery(bq, {
473+
res: {
474+
jobComplete: true,
475+
totalRows: '0',
476+
arrowRecordBatch: {
477+
// An empty Uint8Array is truthy but carries no rows.
478+
serializedRecordBatch: new Uint8Array(0) as unknown as string,
479+
},
480+
},
481+
});
482+
483+
// Verify one 0-row, 0-column RecordBatch is emitted instead of an error.
484+
const batches: arrow.RecordBatch[] = await bq
485+
.createQueryArrowStream('SELECT 1')
486+
.toArray();
487+
assert.strictEqual(batches.length, 1);
488+
assert.strictEqual(batches[0].numRows, 0);
489+
assert.strictEqual(batches[0].numCols, 0);
490+
});
491+
452492
it('should emit a 0-row RecordBatch preserving the schema when arrowSchema is present without arrowRecordBatch', async () => {
453493
// Include nested Struct and List fields to verify 0-row buffer initialization.
454494
const schema = new arrow.Schema([

0 commit comments

Comments
 (0)