Skip to content

Commit bc0126a

Browse files
keshavdandevakeshavdandeva
authored
fix(bigquery-jdbc): format nested ARRAY fields as valid JSON in Struct.toString() (#14720)
b/570669133 ## Summary Fixes `BigQueryBaseStruct.toString()` (and `ResultSet.getString()` on `STRUCT` / `ARRAY<STRUCT>` columns) so that nested `ARRAY` fields inside a `STRUCT` have their elements formatted as valid JSON values instead of unquoted `Arrays.deepToString()` strings. ## Problem `BigQueryBaseStruct.toString()` constructs a JSON object string (`{...}`) and quotes scalar `STRING`, `DATE`, `TIME`, `DATETIME`, `TIMESTAMP`, `GEOGRAPHY`, and Base64-encoded `BYTES` attributes. However, when a `STRUCT` contained a nested `ARRAY` attribute (`BigQueryBaseArray`), `BigQueryBaseStruct.toString()` fell through to `value.toString()`, which called `BigQueryBaseArray.toString()` (`Arrays.deepToString`). This produced unquoted array elements inside the JSON object (e.g., `{"str_arr":[hello, world],"date_arr":[2026-01-01, 2026-01-02]}`), resulting in syntactically invalid JSON. ## Solution - Extracted the existing scalar JSON value formatting logic in `BigQueryBaseStruct.toString()` into a shared helper method `appendJsonValue(...)`. - When a `STRUCT` attribute is a `BigQueryBaseArray`, iterate over its elements and format each element using `appendJsonValue(...)` inside `[...]`. - Top-level `BigQueryBaseArray.toString()` behavior remains unchanged for backwards compatibility. ## Example Output For `SELECT STRUCT(1 AS id, ['hello', 'world'] AS str_arr, [DATE '2026-01-01'] AS date_arr, [TIME '12:34:56.789012'] AS time_arr, [TIMESTAMP '2026-01-01 00:00:00 UTC'] AS ts_arr, [b'abc'] AS bytes_arr)`: - **Before:** `{"id":1,"str_arr":[hello, world],"date_arr":[2026-01-01],"time_arr":[12:34:56.789012],"ts_arr":[2026-01-01 00:00:00.0],"bytes_arr":[YWJj]}` (Invalid JSON) - **After:** `{"id":1,"str_arr":["hello","world"],"date_arr":["2026-01-01"],"time_arr":["12:34:56.789012"],"ts_arr":["2026-01-01 00:00:00.0"],"bytes_arr":["YWJj"]}` (Valid JSON) ## Testing - Added `struct.toString()` assertions covering all BigQuery array types to `BigQueryJsonStructTest#structOfArrays` and `BigQueryArrowStructTest#structOfArrays`. - Added integration test `ITBigQueryJDBCTest#testStructWithNestedArraysValidJson` verifying both REST and Storage Read API (`EnableHighThroughputAPI=1`) paths.
1 parent 13310c7 commit bc0126a

4 files changed

Lines changed: 106 additions & 23 deletions

File tree

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseStruct.java‎

Lines changed: 47 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.sql.Timestamp;
3030
import java.util.Base64;
3131
import java.util.Map;
32+
import org.apache.arrow.vector.util.Text;
3233

3334
/**
3435
* An abstract implementation of {@link java.sql.Struct} used as a base class for {@link
@@ -86,30 +87,24 @@ public String toString() {
8687
Object value = attributes[i];
8788

8889
sb.append("\"").append(fieldName.replace("\"", "\\\"")).append("\":");
89-
90-
if (value == null) {
91-
sb.append("null");
92-
} else if (value instanceof String || value instanceof org.apache.arrow.vector.util.Text) {
93-
String stringValue = value.toString().replace("\"", "\\\"");
94-
sb.append("\"").append(stringValue).append("\"");
95-
} else if (value instanceof Timestamp
96-
&& field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
97-
sb.append("\"")
98-
.append(BigQueryTemporalUtility.formatTimestampToUtcString((Timestamp) value))
99-
.append("\"");
100-
} else if (value instanceof Time
101-
&& field.getType().getStandardType() == StandardSQLTypeName.TIME) {
102-
sb.append("\"")
103-
.append(
104-
BigQueryTypeRegistry.convert(
105-
getRawAttribute(i), StandardSQLTypeName.TIME, String.class, null))
106-
.append("\"");
107-
} else if (value instanceof Timestamp || value instanceof Date || value instanceof Time) {
108-
sb.append("\"").append(value.toString()).append("\"");
109-
} else if (value instanceof byte[]) {
110-
sb.append("\"").append(Base64.getEncoder().encodeToString((byte[]) value)).append("\"");
90+
if (value instanceof BigQueryBaseArray) {
91+
BigQueryBaseArray arrayVal = (BigQueryBaseArray) value;
92+
Object[] elements = (Object[]) arrayVal.getArray();
93+
if (elements == null) {
94+
sb.append("null");
95+
} else {
96+
sb.append("[");
97+
for (int j = 0; j < elements.length; j++) {
98+
if (j > 0) {
99+
sb.append(",");
100+
}
101+
appendJsonValue(
102+
sb, field, elements[j], elements[j] == null ? null : arrayVal.getRawElement(j));
103+
}
104+
sb.append("]");
105+
}
111106
} else {
112-
sb.append(value.toString());
107+
appendJsonValue(sb, field, value, value == null ? null : getRawAttribute(i));
113108
}
114109
}
115110
sb.append("}");
@@ -119,4 +114,33 @@ public String toString() {
119114
return "{ \"error\": \"Error converting struct to string: " + e.getMessage() + "\" }";
120115
}
121116
}
117+
118+
private static void appendJsonValue(StringBuilder sb, Field field, Object value, Object rawValue)
119+
throws SQLException {
120+
if (value == null) {
121+
sb.append("null");
122+
} else if (field.getType().getStandardType() == StandardSQLTypeName.JSON) {
123+
sb.append(value);
124+
} else if (value instanceof String || value instanceof Text) {
125+
String stringValue = value.toString().replace("\"", "\\\"");
126+
sb.append("\"").append(stringValue).append("\"");
127+
} else if (value instanceof Timestamp
128+
&& field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
129+
sb.append("\"")
130+
.append(BigQueryTemporalUtility.formatTimestampToUtcString((Timestamp) value))
131+
.append("\"");
132+
} else if (value instanceof Time
133+
&& field.getType().getStandardType() == StandardSQLTypeName.TIME) {
134+
sb.append("\"")
135+
.append(
136+
BigQueryTypeRegistry.convert(rawValue, StandardSQLTypeName.TIME, String.class, null))
137+
.append("\"");
138+
} else if (value instanceof Timestamp || value instanceof Date || value instanceof Time) {
139+
sb.append("\"").append(value).append("\"");
140+
} else if (value instanceof byte[]) {
141+
sb.append("\"").append(Base64.getEncoder().encodeToString((byte[]) value)).append("\"");
142+
} else {
143+
sb.append(value);
144+
}
145+
}
122146
}

‎java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowStructTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,20 @@ GEOGRAPHY, new Text("POINT(-122 47)"), new Text("POINT(-122 48)")),
181181
.isEqualTo(new String[] {"POINT(-122 47)", "POINT(-122 48)"});
182182
assertThat(((Array) attributes[11]).getArray())
183183
.isEqualTo(new byte[][] {"one".getBytes(), "two".getBytes()});
184+
assertThat(struct.toString())
185+
.isEqualTo(
186+
"{\"INT64_arr_column\":[10,20],"
187+
+ "\"BOOL_arr_column\":[true,false],"
188+
+ "\"FLOAT64_arr_column\":[11.2,33.4],"
189+
+ "\"NUMERIC_arr_column\":[11.2657,33.4657],"
190+
+ "\"BIGNUMERIC_arr_column\":[11.2657,33.4657],"
191+
+ "\"STRING_arr_column\":[\"one\",\"two\"],"
192+
+ "\"TIMESTAMP_arr_column\":[\"2023-03-30 11:14:19.820227\",\"2023-03-31 11:14:19.820227\"],"
193+
+ "\"DATE_arr_column\":[\"2023-03-30\",\"2023-03-31\"],"
194+
+ "\"TIME_arr_column\":[\"11:14:19.820227\",\"11:14:20.820227\"],"
195+
+ "\"DATETIME_arr_column\":[\"2023-03-30 11:14:19.820227\",\"2023-03-30 11:15:19.820227\"],"
196+
+ "\"GEOGRAPHY_arr_column\":[\"POINT(-122 47)\",\"POINT(-122 48)\"],"
197+
+ "\"BYTES_arr_column\":[\"b25l\",\"dHdv\"]}");
184198
}
185199

186200
@Test

‎java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJsonStructTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,20 @@ public void structOfArrays() throws SQLException {
193193
.isEqualTo(new String[] {"POINT(-122 47)", "POINT(-122 48)"});
194194
assertThat(((Array) attributes[11]).getArray())
195195
.isEqualTo(new byte[][] {"one".getBytes(), "two".getBytes()});
196+
assertThat(struct.toString())
197+
.isEqualTo(
198+
"{\"INT64_arr_column\":[10,20],"
199+
+ "\"BOOL_arr_column\":[true,false],"
200+
+ "\"FLOAT64_arr_column\":[11.2,33.4],"
201+
+ "\"NUMERIC_arr_column\":[11.2657,33.4657],"
202+
+ "\"BIGNUMERIC_arr_column\":[11.2657,33.4657],"
203+
+ "\"STRING_arr_column\":[\"one\",\"two\"],"
204+
+ "\"TIMESTAMP_arr_column\":[\"2023-03-30 11:14:19.82\",\"2023-03-31 11:14:19.82\"],"
205+
+ "\"DATE_arr_column\":[\"2023-03-30\",\"2023-03-31\"],"
206+
+ "\"TIME_arr_column\":[\"11:14:19.82\",\"11:14:20.82\"],"
207+
+ "\"DATETIME_arr_column\":[\"2023-03-30 11:14:19.82\",\"2023-03-30 11:15:19.82\"],"
208+
+ "\"GEOGRAPHY_arr_column\":[\"POINT(-122 47)\",\"POINT(-122 48)\"],"
209+
+ "\"BYTES_arr_column\":[\"b25l\",\"dHdv\"]}");
196210
}
197211

198212
@Test

‎java-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITBigQueryJDBCTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@
7171
import org.junit.jupiter.api.BeforeAll;
7272
import org.junit.jupiter.api.Tag;
7373
import org.junit.jupiter.api.Test;
74+
import org.junit.jupiter.params.ParameterizedTest;
75+
import org.junit.jupiter.params.provider.ValueSource;
7476

7577
public class ITBigQueryJDBCTest extends ITBase {
7678
static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
@@ -3095,4 +3097,33 @@ public void testSessionAbortedOnConnectionClose() throws SQLException {
30953097
}
30963098
}
30973099
}
3100+
3101+
@ParameterizedTest
3102+
@ValueSource(strings = {"", FORCE_READ_API_PROPERTIES})
3103+
public void testStructWithNestedArraysValidJson(String extraProps) throws SQLException {
3104+
String sql =
3105+
"SELECT STRUCT("
3106+
+ "1 AS id, "
3107+
+ "['hello', 'world'] AS str_arr, "
3108+
+ "[DATE '2026-01-01', DATE '2026-01-02'] AS date_arr, "
3109+
+ "[TIME '12:34:56.789012'] AS time_arr, "
3110+
+ "[TIMESTAMP '2026-01-01 00:00:00 UTC'] AS ts_arr, "
3111+
+ "[b'abc', b'def'] AS bytes_arr"
3112+
+ ") AS s_nested_arrays";
3113+
3114+
String expectedJson =
3115+
"{\"id\":1,"
3116+
+ "\"str_arr\":[\"hello\",\"world\"],"
3117+
+ "\"date_arr\":[\"2026-01-01\",\"2026-01-02\"],"
3118+
+ "\"time_arr\":[\"12:34:56.789012\"],"
3119+
+ "\"ts_arr\":[\"2026-01-01 00:00:00.0\"],"
3120+
+ "\"bytes_arr\":[\"YWJj\",\"ZGVm\"]}";
3121+
3122+
try (Connection conn = DriverManager.getConnection(connection_uri + extraProps);
3123+
Statement stmt = conn.createStatement();
3124+
ResultSet rs = stmt.executeQuery(sql)) {
3125+
assertTrue(rs.next());
3126+
assertEquals(expectedJson, rs.getString("s_nested_arrays"));
3127+
}
3128+
}
30983129
}

0 commit comments

Comments
 (0)