Skip to content

Commit 929a424

Browse files
authored
feat(bigquery-jdbc): record Statement execution and ConnectionAttempts (#14377)
1 parent 0ae90d5 commit 929a424

13 files changed

Lines changed: 395 additions & 96 deletions

‎java-bigquery-jdbc/README.MD‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -92,27 +92,15 @@ a connection property (`EnableDiagnosticTelemetry=0`), environment variable,
9292
or system property, the service immediately shuts down and remains permanently
9393
off for the remainder of the JVM lifecycle. It will only restart when
9494
the JVM restarts.
95-
* Global Worker Settings: Because the telemetry batcher is a shared global
96-
resource, properties that configure its behavior (`TelemetryUploadInterval`
97-
and `TelemetryBatchSize`) are exclusively evaluated upon establishing the first
98-
connection. Subsequent overrides for these specific properties are ignored.
9995

10096
#### Configuration Keys
10197

10298
To configure these properties, you can use the following keys:
10399

104100
* Opt-Out Control:
105101
* Connection Property: `EnableDiagnosticTelemetry=0;`
106-
* Environment Variable: `GOOGLE_BIGQUERY_JDBC_TELEMETRY_ENABLED=false`
107-
* System Property: `-Dgoogle.bigquery.jdbc.telemetry.enabled=false`
108-
* Upload Interval:
109-
* Connection Property: `TelemetryUploadInterval=300000;`
110-
* Environment Variable: `GOOGLE_BIGQUERY_JDBC_TELEMETRY_INTERVAL_MS=300000`
111-
* System Property: `-Dgoogle.bigquery.jdbc.telemetry.interval_ms=300000`
112-
* Batch Size:
113-
* Connection Property: `TelemetryBatchSize=5000;`
114-
* Environment Variable: `GOOGLE_BIGQUERY_JDBC_TELEMETRY_BATCH_SIZE=5000`
115-
* System Property: `-Dgoogle.bigquery.jdbc.telemetry.batch_size=5000`
102+
* Environment Variable: `BIGQUERY_JDBC_TELEMETRY_ENABLED=false`
103+
* System Property: `-DBIGQUERY_JDBC_TELEMETRY_ENABLED=false`
116104

117105
## Developer Guide
118106

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import com.google.cloud.bigquery.StandardSQLTypeName;
2929
import com.google.cloud.bigquery.exception.BigQueryConversionException;
3030
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
31+
import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
32+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
3133
import io.opentelemetry.api.trace.Span;
3234
import io.opentelemetry.api.trace.SpanContext;
3335
import io.opentelemetry.context.Context;
@@ -258,6 +260,11 @@ public ResultSetMetaData getMetaData() throws SQLException {
258260
metaData = BigQueryResultSetMetadata.of(this.schema.getFields(), this.statement);
259261
}
260262
}
263+
264+
TelemetryManager.recordFeatureUsage(
265+
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
266+
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
267+
261268
return BigQueryJdbcContextProxy.wrap(metaData, ResultSetMetaData.class, connectionId);
262269
}
263270

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
4141
import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException;
4242
import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException;
43+
import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
44+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
4345
import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;
4446
import com.google.cloud.bigquery.storage.v1.BigQueryReadSettings;
4547
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
@@ -483,6 +485,9 @@ public Statement createStatement() throws SQLException {
483485
BigQueryStatement currentStatement = new BigQueryStatement(this);
484486
LOG.fine("Statement %s created.", currentStatement);
485487
addOpenStatements(currentStatement);
488+
489+
TelemetryManager.recordFeatureUsage(
490+
DriverFeature.DRIVER_FEATURE_CUSTOM, "DRIVER_FEATURE_REGULAR_STATEMENT");
486491
return currentStatement;
487492
}
488493

@@ -541,6 +546,9 @@ public PreparedStatement prepareStatement(String sql) throws SQLException {
541546
PreparedStatement currentStatement = new BigQueryPreparedStatement(this, sql);
542547
LOG.fine("Prepared Statement %s created.", currentStatement);
543548
addOpenStatements(currentStatement);
549+
550+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PREPARED_STATEMENT);
551+
544552
return currentStatement;
545553
}
546554

@@ -681,6 +689,8 @@ private void beginTransaction() {
681689
updateSessionInfo(transactionBeginJob.getStatistics().getSessionInfo().getSessionId());
682690
}
683691
this.transactionStarted = true;
692+
693+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_TRANSACTIONS);
684694
} catch (InterruptedException ex) {
685695
throw new BigQueryJdbcRuntimeException("Failed to begin transaction", ex);
686696
}
@@ -915,6 +925,12 @@ public void setAutoCommit(boolean autoCommit) throws SQLException {
915925
if (!this.autoCommit) {
916926
beginTransaction();
917927
}
928+
929+
if (autoCommit) {
930+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_AUTOCOMMIT_ENABLED);
931+
} else {
932+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_AUTOCOMMIT_DISABLED);
933+
}
918934
}
919935

920936
@Override
@@ -972,6 +988,9 @@ public DatabaseMetaData getMetaData() throws SQLException {
972988
if (databaseMetaData == null) {
973989
databaseMetaData = new BigQueryDatabaseMetaData(this);
974990
}
991+
992+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL);
993+
975994
return databaseMetaData;
976995
}
977996

@@ -1474,6 +1493,9 @@ public CallableStatement prepareCall(String sql) throws SQLException {
14741493
CallableStatement currentStatement = new BigQueryCallableStatement(this, sql);
14751494
LOG.fine("Callable Statement %s created.", currentStatement);
14761495
addOpenStatements(currentStatement);
1496+
1497+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_CALLABLE_STATEMENT);
1498+
14771499
return currentStatement;
14781500
}
14791501

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryDriver.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@
1818

1919
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
2020
import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException;
21+
import com.google.cloud.bigquery.jdbc.telemetry.v1.AuthenticationType;
22+
import com.google.cloud.bigquery.jdbc.telemetry.v1.Status;
23+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
2124
import com.google.cloud.bigquery.jdbc.utils.BigQueryJdbcVersionUtility;
2225
import io.grpc.LoadBalancerRegistry;
2326
import io.grpc.internal.PickFirstLoadBalancerProvider;
@@ -124,6 +127,7 @@ public static BigQueryDriver getRegisteredDriver() throws IllegalStateException
124127
@Override
125128
public Connection connect(String url, Properties info) throws SQLException {
126129
LOG.finest("++enter++");
130+
AuthenticationType authType = AuthenticationType.AUTHENTICATION_TYPE_UNSPECIFIED;
127131
try {
128132
if (acceptsURL(url)) {
129133
Properties connectInfo = info == null ? new Properties() : (Properties) info.clone();
@@ -132,6 +136,17 @@ public Connection connect(String url, Properties info) throws SQLException {
132136
String connectionUri =
133137
BigQueryJdbcUrlUtility.appendPropertiesToURL(
134138
url.substring(5), this.toString(), connectInfo);
139+
140+
String telemetryOptOut =
141+
BigQueryJdbcUrlUtility.parseUriPropertyWithoutValidation(
142+
connectionUri, BigQueryJdbcUrlUtility.ENABLE_DIAGNOSTIC_TELEMETRY_PROPERTY_NAME);
143+
144+
if (telemetryOptOut != null) {
145+
connectInfo.setProperty(
146+
BigQueryJdbcUrlUtility.ENABLE_DIAGNOSTIC_TELEMETRY_PROPERTY_NAME, telemetryOptOut);
147+
}
148+
TelemetryManager.getInstance(connectInfo);
149+
135150
Level logLevel;
136151
String logPath;
137152
try {
@@ -200,14 +215,25 @@ public Connection connect(String url, Properties info) throws SQLException {
200215
logLevel,
201216
logPath,
202217
this.toString());
203-
return BigQueryJdbcContextProxy.wrap(connection, Connection.class);
218+
219+
Connection wrapped = BigQueryJdbcContextProxy.wrap(connection, Connection.class);
220+
221+
authType = TelemetryManager.toAuthenticationType(ds.getOAuthType());
222+
TelemetryManager.recordConnectionAttempt(Status.STATUS_SUCCESS, 0, authType);
223+
return wrapped;
204224
} else {
205225
return null;
206226
}
207227
} catch (IOException e) {
208-
LOG.warning("Getting a warning: " + e.getMessage());
228+
TelemetryManager.recordConnectionAttempt(
229+
Status.STATUS_ERROR, TelemetryManager.extractErrorCode(e), authType);
230+
LOG.warning("Getting a warning: %s", e.getMessage());
231+
return null;
232+
} catch (Throwable t) {
233+
TelemetryManager.recordConnectionAttempt(
234+
Status.STATUS_ERROR, TelemetryManager.extractErrorCode(t), authType);
235+
throw t;
209236
}
210-
return null;
211237
}
212238

213239
/**

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryParameterHandler.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import com.google.cloud.bigquery.StandardSQLTypeName;
2222
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
2323
import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException;
24+
import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
25+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
2426
import java.math.BigInteger;
2527
import java.sql.SQLException;
2628
import java.sql.Time;
@@ -157,6 +159,8 @@ void setParameter(int parameterIndex, Object value, Class type)
157159
parameter.setParamType(BigQueryStatementParameterType.UNSPECIFIED);
158160
parameter.setScale(-1);
159161

162+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
163+
160164
LOG.finest("Parameter set { %s }", parameter.toString());
161165
}
162166

@@ -239,6 +243,9 @@ void setParameter(
239243
if (parameter.getIndex() == -1) {
240244
parametersList.add(parameter);
241245
}
246+
247+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
248+
242249
LOG.finest("Parameter set { %s }", parameter.toString());
243250
}
244251

@@ -272,6 +279,8 @@ void setParameter(
272279
parameter.setParamType(paramType);
273280
parameter.setScale(scale);
274281

282+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_PARAMETER_BINDING);
283+
275284
LOG.finest("Parameter set { %s }", parameter.toString());
276285
}
277286

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPooledConnection.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,11 @@
1616

1717
package com.google.cloud.bigquery.jdbc;
1818

19+
import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
20+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
1921
import com.google.common.annotations.VisibleForTesting;
2022
import java.sql.Connection;
23+
import java.sql.DatabaseMetaData;
2124
import java.sql.SQLException;
2225
import java.util.UUID;
2326
import java.util.concurrent.Executor;
@@ -231,7 +234,8 @@ public void rollback() throws SQLException {
231234
}
232235

233236
@Override
234-
public java.sql.DatabaseMetaData getMetaData() throws SQLException {
237+
public DatabaseMetaData getMetaData() throws SQLException {
238+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL);
235239
return bqConnectionDelegate.getMetaData();
236240
}
237241

‎java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryPreparedStatement.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import com.google.cloud.bigquery.exception.BigQueryJdbcException;
2929
import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException;
3030
import com.google.cloud.bigquery.exception.BigQueryJdbcSqlFeatureNotSupportedException;
31+
import com.google.cloud.bigquery.jdbc.telemetry.v1.DriverFeature;
32+
import com.google.cloud.bigquery.jdbc.telemetry.v1.StatementExecution;
33+
import com.google.cloud.bigquery.jdbc.telemetry.v1.TelemetryManager;
3134
import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsRequest;
3235
import com.google.cloud.bigquery.storage.v1.BatchCommitWriteStreamsResponse;
3336
import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
@@ -323,7 +326,16 @@ public int[] executeBatch() throws SQLException {
323326
if (this.batchParameters.isEmpty()) {
324327
return result;
325328
}
329+
326330
if (useWriteAPI()) {
331+
long startTime = System.currentTimeMillis();
332+
StatementExecution.Builder writeApiExecutionBuilder =
333+
StatementExecution.newBuilder()
334+
.setStatementType(
335+
com.google.cloud.bigquery.jdbc.telemetry.v1.StatementType.STATEMENT_TYPE_INSERT)
336+
.setQueryApiType(
337+
com.google.cloud.bigquery.jdbc.telemetry.v1.QueryApiType
338+
.QUERY_API_TYPE_WRITE_API);
327339
try (BigQueryWriteClient writeClient = this.connection.getBigQueryWriteClient()) {
328340
LOG.info("Using Write API for bulk INSERT operation.");
329341
ArrayList<BigQueryJdbcParameter> currentParameterList = this.batchParameters.peek();
@@ -336,10 +348,24 @@ public int[] executeBatch() throws SQLException {
336348
long rowCount = bulkInsertWithWriteAPI(writeClient);
337349
int[] insertArray = new int[Math.toIntExact(rowCount)];
338350
Arrays.fill(insertArray, 1);
351+
352+
writeApiExecutionBuilder.setStatus(
353+
com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_SUCCESS);
354+
339355
return insertArray;
340356

341357
} catch (DescriptorValidationException | IOException | InterruptedException e) {
358+
writeApiExecutionBuilder
359+
.setStatus(com.google.cloud.bigquery.jdbc.telemetry.v1.Status.STATUS_ERROR)
360+
.setErrorCode(TelemetryManager.extractErrorCode(e));
361+
if (e instanceof InterruptedException) {
362+
Thread.currentThread().interrupt();
363+
}
342364
throw new BigQueryJdbcRuntimeException("Failed to execute batch with Write API", e);
365+
} finally {
366+
long durationMs = System.currentTimeMillis() - startTime;
367+
TelemetryManager.recordStatementExecution(writeApiExecutionBuilder, durationMs);
368+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);
343369
}
344370

345371
} else {
@@ -370,6 +396,8 @@ public int[] executeBatch() throws SQLException {
370396
throw new BigQueryJdbcRuntimeException("Interrupted during individual INSERT batch", ex);
371397
} catch (SQLException e) {
372398
throw new BigQueryJdbcException("SQL error during individual INSERT batch", e);
399+
} finally {
400+
TelemetryManager.recordFeatureUsage(DriverFeature.DRIVER_FEATURE_BATCH_OPERATIONS);
373401
}
374402
}
375403
}
@@ -542,6 +570,11 @@ public ResultSetMetaData getMetaData() throws SQLException {
542570
if (this.insertSchema != null) {
543571
return BigQueryResultSetMetadata.of(this.insertSchema.getFields(), this);
544572
}
573+
574+
TelemetryManager.recordFeatureUsage(
575+
DriverFeature.DRIVER_FEATURE_METADATA_RETRIEVAL,
576+
"DRIVER_FEATURE_RESULTSET_METADATA_RETRIEVAL");
577+
545578
return null;
546579
}
547580

0 commit comments

Comments
 (0)