Skip to content

Commit 66c58d3

Browse files
fix(bigquery): apply DataFormatOptions to tabledata.list requests (#14889)
b/579291912 `BigQueryOptions.setDataFormatOptions(...)` is only sent on `jobs.query`. Rows read through `tabledata.list` don't get it, so BigQuery returns TIMESTAMPs as float64 there while the client decodes them using the configured format. `tabledata.list` serves: - `listTableData` / `Table.list` - page 2+ of `query()` - `Job.getQueryResults()` Impact: - `useInt64Timestamp(true)`: `getTimestampValue()` / `getTimestampInstant()` throw `NumberFormatException`. - `timestampFormatOptions(INT64)`: values are decoded as seconds, so dates come out wrong. This also happens on page 1. - `ISO8601_STRING`: rows from `tabledata.list` come back as float64. - float64 loses microseconds, e.g. `9999-12-31 23:59:59.999999` becomes `+10000-01-01`. ## Changes - **`HttpBigQueryRpc`**: `listTableData` sends `formatOptions.useInt64Timestamp` / `formatOptions.timestampOutputFormat`, only when they differ from the defaults. Default requests are unchanged. - **`BigQueryImpl`**: rows are decoded as int64 when either `useInt64Timestamp(true)` or `timestampFormatOptions(INT64)` is set. No public API changes. ## Tests - `HttpBigQueryRpcTest`: defaults send no `formatOptions`, and configured options are sent. - `BigQueryImplTest`: `INT64` values from `listTableData` decode as int64. - `ITBigQueryTest#testLosslessMaxTimestampIntegration`: now reads 2 pages (the second via `tabledata.list`) and adds an `INT64` case. All new and changed tests fail without the fix and pass with it.
1 parent aa38063 commit 66c58d3

5 files changed

Lines changed: 99 additions & 17 deletions

File tree

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2213,18 +2213,19 @@ public TableDataList call() throws IOException {
22132213
new PageImpl<>(
22142214
new TableDataPageFetcher(tableId, schema, serviceOptions, cursor, pageOptionMap),
22152215
cursor,
2216-
transformTableData(
2217-
result.getRows(),
2218-
schema,
2219-
serviceOptions.getDataFormatOptions().useInt64Timestamp())),
2216+
transformTableData(result.getRows(), schema, serviceOptions.getDataFormatOptions())),
22202217
result.getTotalRows());
22212218
} catch (BigQueryRetryHelperException e) {
22222219
throw BigQueryException.translateAndThrow(e);
22232220
}
22242221
}
22252222

22262223
private static Iterable<FieldValueList> transformTableData(
2227-
Iterable<TableRow> tableDataPb, final Schema schema, boolean useInt64Timestamps) {
2224+
Iterable<TableRow> tableDataPb, final Schema schema, DataFormatOptions dataFormatOptions) {
2225+
final boolean useInt64Timestamps =
2226+
dataFormatOptions.useInt64Timestamp()
2227+
|| dataFormatOptions.timestampFormatOptions()
2228+
== DataFormatOptions.TimestampFormatOptions.INT64;
22282229
return ImmutableList.copyOf(
22292230
Iterables.transform(
22302231
tableDataPb != null ? tableDataPb : ImmutableList.<TableRow>of(),
@@ -2490,9 +2491,7 @@ public com.google.api.services.bigquery.model.QueryResponse call()
24902491
new QueryPageFetcher(jobId, schema, getOptions(), cursor, optionMap(options)),
24912492
cursor,
24922493
transformTableData(
2493-
results.getRows(),
2494-
schema,
2495-
getOptions().getDataFormatOptions().useInt64Timestamp())))
2494+
results.getRows(), schema, getOptions().getDataFormatOptions())))
24962495
.setJobId(jobId)
24972496
.setRowsInPage(results.getRows() != null ? (long) results.getRows().size() : 0L)
24982497
.build();
@@ -2504,10 +2503,7 @@ public com.google.api.services.bigquery.model.QueryResponse call()
25042503
new PageImpl<>(
25052504
new TableDataPageFetcher(null, schema, getOptions(), null, optionMap(options)),
25062505
null,
2507-
transformTableData(
2508-
results.getRows(),
2509-
schema,
2510-
getOptions().getDataFormatOptions().useInt64Timestamp())))
2506+
transformTableData(results.getRows(), schema, getOptions().getDataFormatOptions())))
25112507
// Return the JobID of the successful job
25122508
.setJobId(
25132509
results.getJobReference() != null ? JobId.fromPb(results.getJobReference()) : null)

‎java-bigquery/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpc.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
import com.google.cloud.Tuple;
6868
import com.google.cloud.bigquery.BigQueryException;
6969
import com.google.cloud.bigquery.BigQueryOptions;
70+
import com.google.cloud.bigquery.DataFormatOptions;
71+
import com.google.cloud.bigquery.DataFormatOptions.TimestampFormatOptions;
7072
import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer;
7173
import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer;
7274
import com.google.cloud.http.HttpTransportOptions;
@@ -1419,6 +1421,8 @@ public TableDataList listTableDataSkipExceptionTranslation(
14191421
String projectId, String datasetId, String tableId, Map<Option, ?> options)
14201422
throws IOException {
14211423
validateRPC();
1424+
DataFormatOptions formatOptions = this.options.getDataFormatOptions();
1425+
TimestampFormatOptions timestampFormat = formatOptions.timestampFormatOptions();
14221426
Bigquery.Tabledata.List bqListRequest =
14231427
bigquery
14241428
.tabledata()
@@ -1429,6 +1433,11 @@ public TableDataList listTableDataSkipExceptionTranslation(
14291433
.setStartIndex(
14301434
Option.START_INDEX.getLong(options) != null
14311435
? BigInteger.valueOf(Option.START_INDEX.getLong(options))
1436+
: null)
1437+
.setFormatOptionsUseInt64Timestamp(formatOptions.useInt64Timestamp() ? true : null)
1438+
.setFormatOptionsTimestampOutputFormat(
1439+
timestampFormat != TimestampFormatOptions.TIMESTAMP_OUTPUT_FORMAT_UNSPECIFIED
1440+
? timestampFormat.toString()
14321441
: null);
14331442

14341443
bqListRequest

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1783,6 +1783,34 @@ void testListTableData() throws IOException {
17831783
.listTableDataSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS);
17841784
}
17851785

1786+
@Test
1787+
void testListTableDataWithInt64TimestampFormat() throws IOException {
1788+
TableDataList tableDataPb =
1789+
new TableDataList()
1790+
.setTotalRows(1L)
1791+
.setRows(
1792+
ImmutableList.of(
1793+
new TableRow()
1794+
.setF(ImmutableList.of(new TableCell().setV("253402300799999999")))));
1795+
when(bigqueryRpcMock.listTableDataSkipExceptionTranslation(
1796+
PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS))
1797+
.thenReturn(tableDataPb);
1798+
bigquery =
1799+
options.toBuilder()
1800+
.setDataFormatOptions(
1801+
DataFormatOptions.newBuilder()
1802+
.timestampFormatOptions(DataFormatOptions.TimestampFormatOptions.INT64)
1803+
.build())
1804+
.build()
1805+
.getService();
1806+
1807+
Page<FieldValueList> page = bigquery.listTableData(DATASET, TABLE);
1808+
assertEquals(
1809+
253402300799999999L, page.getValues().iterator().next().get(0).getTimestampValue());
1810+
verify(bigqueryRpcMock)
1811+
.listTableDataSkipExceptionTranslation(PROJECT, DATASET, TABLE, EMPTY_RPC_OPTIONS);
1812+
}
1813+
17861814
@Test
17871815
void testListTableDataFromTableId() throws IOException {
17881816
when(bigqueryRpcMock.listTableDataSkipExceptionTranslation(

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,8 +1236,12 @@ static GoogleCredentials loadCredentials(String credentialFile) {
12361236

12371237
@Test
12381238
void testLosslessMaxTimestampIntegration() throws InterruptedException {
1239-
String query = "SELECT TIMESTAMP '9999-12-31 23:59:59.999999 UTC' as max_ts";
1240-
QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).build();
1239+
// Two rows, one per page, so the second page is read through tabledata.list.
1240+
String query =
1241+
"SELECT max_ts FROM UNNEST([TIMESTAMP '9999-12-31 23:59:59.999999 UTC',"
1242+
+ " TIMESTAMP '9999-12-31 23:59:59.999999 UTC']) AS max_ts";
1243+
QueryJobConfiguration config =
1244+
QueryJobConfiguration.newBuilder(query).setMaxResults(1L).build();
12411245

12421246
// 1. Test lossless 64-bit integer parsing (useInt64Timestamps = true)
12431247
DataFormatOptions losslessOptions =
@@ -1249,7 +1253,7 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException {
12491253
.getService();
12501254

12511255
TableResult losslessResult = losslessBigQuery.query(config);
1252-
assertEquals(1L, losslessResult.getTotalRows());
1256+
assertEquals(2L, losslessResult.getTotalRows());
12531257
for (FieldValueList row : losslessResult.iterateAll()) {
12541258
long exactMicros = row.get("max_ts").getTimestampValue();
12551259
assertEquals(253402300799999999L, exactMicros);
@@ -1262,7 +1266,7 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException {
12621266
bigquery.getOptions().toBuilder().setDataFormatOptions(floatOptions).build().getService();
12631267

12641268
TableResult floatResult = floatBigQuery.query(config);
1265-
assertEquals(1L, floatResult.getTotalRows());
1269+
assertEquals(2L, floatResult.getTotalRows());
12661270
for (FieldValueList row : floatResult.iterateAll()) {
12671271
long roundedMicros = row.get("max_ts").getTimestampValue();
12681272
assertEquals(253402300800000000L, roundedMicros);
@@ -1277,11 +1281,26 @@ void testLosslessMaxTimestampIntegration() throws InterruptedException {
12771281
bigquery.getOptions().toBuilder().setDataFormatOptions(isoOptions).build().getService();
12781282

12791283
TableResult isoResult = isoBigQuery.query(config);
1280-
assertEquals(1L, isoResult.getTotalRows());
1284+
assertEquals(2L, isoResult.getTotalRows());
12811285
for (FieldValueList row : isoResult.iterateAll()) {
12821286
String isoValue = row.get("max_ts").getStringValue();
12831287
assertEquals("9999-12-31T23:59:59.999999Z", isoValue);
12841288
}
1289+
1290+
// 4. Test INT64 timestamp output format
1291+
DataFormatOptions int64Options =
1292+
DataFormatOptions.newBuilder()
1293+
.timestampFormatOptions(DataFormatOptions.TimestampFormatOptions.INT64)
1294+
.build();
1295+
BigQuery int64BigQuery =
1296+
bigquery.getOptions().toBuilder().setDataFormatOptions(int64Options).build().getService();
1297+
1298+
TableResult int64Result = int64BigQuery.query(config);
1299+
assertEquals(2L, int64Result.getTotalRows());
1300+
for (FieldValueList row : int64Result.iterateAll()) {
1301+
long exactMicros = row.get("max_ts").getTimestampValue();
1302+
assertEquals(253402300799999999L, exactMicros);
1303+
}
12851304
}
12861305

12871306
@Test

‎java-bigquery/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/spi/v2/HttpBigQueryRpcTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
import com.google.api.services.bigquery.model.TableReference;
4848
import com.google.cloud.NoCredentials;
4949
import com.google.cloud.bigquery.BigQueryOptions;
50+
import com.google.cloud.bigquery.DataFormatOptions;
51+
import com.google.cloud.bigquery.DataFormatOptions.TimestampFormatOptions;
5052
import com.google.cloud.bigquery.telemetry.BigQueryTelemetryTracer;
5153
import com.google.cloud.bigquery.telemetry.HttpTracingRequestInitializer;
5254
import io.opentelemetry.api.common.AttributeKey;
@@ -1565,9 +1567,37 @@ public void testListTableDataNoTelemetry() throws Exception {
15651567
verifyRequest(
15661568
"GET",
15671569
"/projects/" + PROJECT_ID + "/datasets/" + DATASET_ID + "/tables/" + TABLE_ID + "/data");
1570+
assertThat(lastRequestUrl).doesNotContain("formatOptions");
15681571
verifyNoSpans();
15691572
}
15701573

1574+
@Test
1575+
public void testListTableDataWithDataFormatOptions() throws Exception {
1576+
setMockResponse("{\"kind\":\"bigquery#tableDataList\",\"rows\":[]}");
1577+
1578+
BigQueryOptions customOptions =
1579+
BigQueryOptions.newBuilder()
1580+
.setProjectId(PROJECT_ID)
1581+
.setCredentials(NoCredentials.getInstance())
1582+
.setTransportOptions(
1583+
BigQueryOptions.getDefaultHttpTransportOptions().toBuilder()
1584+
.setHttpTransportFactory(() -> mockTransport)
1585+
.build())
1586+
.setDataFormatOptions(
1587+
DataFormatOptions.newBuilder()
1588+
.useInt64Timestamp(true)
1589+
.timestampFormatOptions(TimestampFormatOptions.ISO8601_STRING)
1590+
.build())
1591+
.build();
1592+
HttpBigQueryRpc customRpc = new HttpBigQueryRpc(customOptions);
1593+
1594+
customRpc.listTableDataSkipExceptionTranslation(
1595+
PROJECT_ID, DATASET_ID, TABLE_ID, new HashMap<>());
1596+
1597+
assertThat(lastRequestUrl).contains("formatOptions.useInt64Timestamp=true");
1598+
assertThat(lastRequestUrl).contains("formatOptions.timestampOutputFormat=ISO8601_STRING");
1599+
}
1600+
15711601
@Test
15721602
public void testListTableDataWithRowLimitNoTelemetry() throws Exception {
15731603
setMockResponse("{\"kind\":\"bigquery#tableDataList\",\"rows\":[]}");

0 commit comments

Comments
 (0)