Skip to content

Commit e1c9326

Browse files
test(spanner): unflake ReadBenchmark by isolating mock server and fixing heap leak (#14417)
- Add `StandaloneBenchmarkServer` and update `ReadBenchmark` (along with the Maven benchmark profile) to connect to an external gRPC mock server when `jmh.spanner.server.port` or `SPANNER_PORT` is specified. This allows running the mock server and JMH client in separate JVMs pinned to disjoint CPU cores. - Add `MockSpannerServiceImpl.setRecordRequests(boolean)` to bypass global synchronized (lock) contention and prevent unbounded retention of 20+ million request protobufs (~21.5 GB Old-Gen heap leak) during 20-minute benchmark runs. - Fix `BenchmarkValidator` so latency improvements (actual < baseline) do not fail validation, and include explicit units (us/op), allowed ceiling, percentage delta, and host SAR CPU telemetry (avg_busy, avg_steal, max_steal) in failure messages.
1 parent b25100d commit e1c9326

7 files changed

Lines changed: 472 additions & 97 deletions

File tree

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MockSpannerServiceImpl.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -851,7 +851,16 @@ public void freezeAfter(int numRequests) {
851851
freezeAfterNumRequests.set(numRequests);
852852
}
853853

854+
private volatile boolean recordRequests = true;
855+
856+
public void setRecordRequests(boolean recordRequests) {
857+
this.recordRequests = recordRequests;
858+
}
859+
854860
private void maybeFreezeAndRecordRequest(AbstractMessage request) {
861+
if (!recordRequests) {
862+
return;
863+
}
855864
synchronized (lock) {
856865
if (freezeAfterNumRequests.get() >= 0) {
857866
if (freezeAfterNumRequests.decrementAndGet() == -1) {

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/BenchmarkValidator.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ void validate() {
6161
double difference =
6262
calculatePercentageDifference(
6363
expectedPercentile.baseline, actualPercentilesMap.get(percentile));
64-
// if an absolute different in percentage is greater than allowed difference
64+
// if a positive difference in percentage is greater than allowed difference
6565
// Then we are throwing validation error
66-
if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) {
66+
if (difference > expectedPercentile.difference) {
6767
throw new ValidationException(
6868
String.format(
6969
"[%s][%s] Expected percentile %s[+/-%s] but got %s",
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.benchmarking;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
import static org.junit.Assert.assertThrows;
21+
22+
import com.google.cloud.spanner.benchmarking.BenchmarkValidator.ValidationException;
23+
import java.io.File;
24+
import java.io.IOException;
25+
import java.nio.charset.StandardCharsets;
26+
import java.nio.file.Files;
27+
import org.junit.Rule;
28+
import org.junit.Test;
29+
import org.junit.rules.TemporaryFolder;
30+
import org.junit.runner.RunWith;
31+
import org.junit.runners.JUnit4;
32+
33+
@RunWith(JUnit4.class)
34+
public class BenchmarkValidatorTest {
35+
36+
@Rule public TemporaryFolder tempFolder = new TemporaryFolder();
37+
38+
@Test
39+
public void validatePassesWhenActualLatencyIsFasterThanBaseline() throws IOException {
40+
File actualFile = tempFolder.newFile("jmh-faster-result.json");
41+
String fasterJson =
42+
"[\n"
43+
+ " {\n"
44+
+ " \"benchmark\":"
45+
+ " \"com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark\",\n"
46+
+ " \"primaryMetric\": {\n"
47+
+ " \"scorePercentiles\": {\n"
48+
+ " \"50.0\": 320.0,\n"
49+
+ " \"95.0\": 600.0,\n"
50+
+ " \"99.0\": 900.0\n"
51+
+ " }\n"
52+
+ " }\n"
53+
+ " }\n"
54+
+ "]";
55+
Files.write(actualFile.toPath(), fasterJson.getBytes(StandardCharsets.UTF_8));
56+
57+
BenchmarkValidator validator =
58+
new BenchmarkValidator(
59+
"com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile.getAbsolutePath());
60+
61+
// Should not throw when latency improves by >20% (320.0 us/op vs 450.0 us/op baseline)
62+
validator.validate();
63+
}
64+
65+
@Test
66+
public void validateThrowsWhenLatencyRegressesBeyondThreshold() throws IOException {
67+
File actualFile = tempFolder.newFile("jmh-slower-result.json");
68+
String slowerJson =
69+
"[\n"
70+
+ " {\n"
71+
+ " \"benchmark\":"
72+
+ " \"com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark\",\n"
73+
+ " \"primaryMetric\": {\n"
74+
+ " \"scorePercentiles\": {\n"
75+
+ " \"50.0\": 629.76,\n"
76+
+ " \"95.0\": 800.0,\n"
77+
+ " \"99.0\": 1200.0\n"
78+
+ " }\n"
79+
+ " }\n"
80+
+ " }\n"
81+
+ "]";
82+
Files.write(actualFile.toPath(), slowerJson.getBytes(StandardCharsets.UTF_8));
83+
84+
BenchmarkValidator validator =
85+
new BenchmarkValidator(
86+
"com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile.getAbsolutePath());
87+
88+
ValidationException ex = assertThrows(ValidationException.class, validator::validate);
89+
assertThat(ex.getMessage())
90+
.contains(
91+
"[com.google.cloud.spanner.benchmarking.ReadBenchmark.queryBenchmark][50.0] Expected"
92+
+ " percentile 450.0[+/-20.0] but got 629.76");
93+
}
94+
}

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/benchmarking/ReadBenchmark.java‎

Lines changed: 30 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -21,26 +21,15 @@
2121
import com.google.cloud.spanner.DatabaseId;
2222
import com.google.cloud.spanner.Key;
2323
import com.google.cloud.spanner.KeySet;
24-
import com.google.cloud.spanner.MockSpannerServiceImpl;
25-
import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult;
2624
import com.google.cloud.spanner.ReadContext;
2725
import com.google.cloud.spanner.ResultSet;
2826
import com.google.cloud.spanner.Spanner;
2927
import com.google.cloud.spanner.SpannerOptions;
3028
import com.google.cloud.spanner.Statement;
31-
import com.google.protobuf.ListValue;
32-
import com.google.spanner.v1.ResultSetMetadata;
33-
import com.google.spanner.v1.StructType;
34-
import com.google.spanner.v1.StructType.Field;
35-
import com.google.spanner.v1.TypeCode;
3629
import io.grpc.ManagedChannelBuilder;
37-
import io.grpc.Server;
38-
import io.grpc.ServerBuilder;
3930
import java.io.IOException;
4031
import java.util.Arrays;
4132
import java.util.List;
42-
import java.util.concurrent.ExecutorService;
43-
import java.util.concurrent.Executors;
4433
import java.util.concurrent.TimeUnit;
4534
import org.openjdk.jmh.annotations.Benchmark;
4635
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -76,113 +65,53 @@ public static class BenchmarkState {
7665
Spanner spanner;
7766
DatabaseClient databaseClient;
7867

79-
// gRPC server
80-
Server gRPCServer;
81-
Server gRPCMonitoringServer;
82-
83-
// Executors for handling parallel requests by gRPC server
84-
ExecutorService gRPCServerExecutor;
85-
8668
// Table
8769
List<String> columns = Arrays.asList("id", "name");
8870
String selectQuery = "SELECT * FROM [TABLE] WHERE ID = 1";
8971

9072
@Setup(Level.Trial)
9173
public void setup() throws IOException {
92-
// Enable JMH system property
93-
System.setProperty("jmh.enabled", "true");
94-
95-
// Initializing mock spanner service
96-
MockSpannerServiceImpl mockSpannerService = new MockSpannerServiceImpl();
97-
mockSpannerService.setAbortProbability(0.0D);
98-
99-
// Initializing mock monitoring service
100-
MonitoringServiceImpl mockMonitoringService = new MonitoringServiceImpl();
101-
102-
// Create a thread pool to handle concurrent requests
103-
gRPCServerExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
74+
setup(System.getenv("SPANNER_PORT"), System.getenv("MONITORING_PORT"));
75+
}
10476

105-
// Creating Spanner Inprocess gRPC server
106-
gRPCServer =
107-
ServerBuilder.forPort(0)
108-
.addService(mockSpannerService)
109-
.executor(gRPCServerExecutor)
110-
.build()
111-
.start();
77+
void setup(String spannerPort, String monitoringPort) throws IOException {
78+
if (spannerPort == null || spannerPort.trim().isEmpty()) {
79+
throw new IllegalStateException(
80+
"SPANNER_PORT environment variable must be specified. Start StandaloneBenchmarkServer"
81+
+ " first.");
82+
}
11283

113-
registerMocks(mockSpannerService);
84+
// Enable JMH system property
85+
System.setProperty("jmh.enabled", "true");
11486

115-
// Creating Monitoring Inprocess gRPC server
116-
gRPCMonitoringServer =
117-
ServerBuilder.forPort(0).addService(mockMonitoringService).build().start();
87+
if (monitoringPort != null && !monitoringPort.trim().isEmpty()) {
88+
// Set the monitoring host port for exporter to forward requests to local netty gRPC server
89+
System.setProperty("jmh.monitoring-server-port", monitoringPort.trim());
90+
}
11891

119-
// Set the monitoring host port for exporter to forward requests to local netty gRPC server
120-
System.setProperty(
121-
"jmh.monitoring-server-port", String.valueOf(gRPCMonitoringServer.getPort()));
92+
final int targetSpannerPort = Integer.parseInt(spannerPort.trim());
12293

12394
spanner =
12495
SpannerOptions.newBuilder()
12596
.setProjectId("[PROJECT]")
97+
.setEmulatorHost(null)
12698
.setCredentials(NoCredentials.getInstance())
12799
.setChannelConfigurator(
128100
managedChannelBuilder ->
129-
ManagedChannelBuilder.forAddress("0.0.0.0", gRPCServer.getPort())
130-
.usePlaintext())
101+
ManagedChannelBuilder.forAddress("0.0.0.0", targetSpannerPort).usePlaintext())
131102
.build()
132103
.getService();
133104
databaseClient =
134105
spanner.getDatabaseClient(DatabaseId.of("[PROJECT]", "[INSTANCE_ID]", "[DATABASE_ID]"));
135106
}
136107

137-
private void registerMocks(MockSpannerServiceImpl mockSpannerService) {
138-
ResultSetMetadata selectMetadata =
139-
ResultSetMetadata.newBuilder()
140-
.setRowType(
141-
StructType.newBuilder()
142-
.addFields(
143-
Field.newBuilder()
144-
.setName("id")
145-
.setType(
146-
com.google.spanner.v1.Type.newBuilder()
147-
.setCode(TypeCode.INT64)
148-
.build())
149-
.build())
150-
.addFields(
151-
Field.newBuilder()
152-
.setName("name")
153-
.setType(
154-
com.google.spanner.v1.Type.newBuilder()
155-
.setCode(TypeCode.STRING)
156-
.build())
157-
.build())
158-
.build())
159-
.build();
160-
com.google.spanner.v1.ResultSet selectResultSet =
161-
com.google.spanner.v1.ResultSet.newBuilder()
162-
.addRows(
163-
ListValue.newBuilder()
164-
.addValues(com.google.protobuf.Value.newBuilder().setStringValue("1").build())
165-
.addValues(
166-
com.google.protobuf.Value.newBuilder().setStringValue("[NAME]").build())
167-
.build())
168-
.setMetadata(selectMetadata)
169-
.build();
170-
mockSpannerService.putStatementResult(
171-
StatementResult.read(
172-
"[TABLE]", KeySet.singleKey(Key.of()), this.columns, selectResultSet));
173-
mockSpannerService.putStatementResult(
174-
StatementResult.query(Statement.of(this.selectQuery), selectResultSet));
175-
}
176-
177108
@TearDown(Level.Trial)
178109
public void tearDown() throws InterruptedException {
179-
spanner.close();
180-
gRPCServer.shutdown();
181-
gRPCServerExecutor.shutdown();
182-
183-
// awaiting termination for servers and executors
184-
gRPCServer.awaitTermination(10, TimeUnit.SECONDS);
185-
gRPCServerExecutor.awaitTermination(10, TimeUnit.SECONDS);
110+
System.clearProperty("jmh.enabled");
111+
System.clearProperty("jmh.monitoring-server-port");
112+
if (spanner != null) {
113+
spanner.close();
114+
}
186115
}
187116
}
188117

@@ -195,7 +124,10 @@ public void readBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) {
195124
try (ResultSet resultSet =
196125
readContext.read("[TABLE]", KeySet.singleKey(Key.of("2")), benchmarkState.columns)) {
197126
while (resultSet.next()) {
198-
blackhole.consume(resultSet.getLong("id"));
127+
long id = resultSet.getLong("id");
128+
if (blackhole != null) {
129+
blackhole.consume(id);
130+
}
199131
}
200132
}
201133
}
@@ -210,7 +142,10 @@ public void queryBenchmark(BenchmarkState benchmarkState, Blackhole blackhole) {
210142
try (ResultSet resultSet =
211143
readContext.executeQuery(Statement.of(benchmarkState.selectQuery))) {
212144
while (resultSet.next()) {
213-
blackhole.consume(resultSet.getLong("id"));
145+
long id = resultSet.getLong("id");
146+
if (blackhole != null) {
147+
blackhole.consume(id);
148+
}
214149
}
215150
}
216151
}
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright 2026 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.benchmarking;
18+
19+
import static com.google.common.truth.Truth.assertThat;
20+
21+
import com.google.cloud.spanner.benchmarking.ReadBenchmark.BenchmarkState;
22+
import com.google.spanner.v1.ExecuteSqlRequest;
23+
import com.google.spanner.v1.ReadRequest;
24+
import org.junit.Test;
25+
import org.junit.runner.RunWith;
26+
import org.junit.runners.JUnit4;
27+
28+
@RunWith(JUnit4.class)
29+
public class ReadBenchmarkTest {
30+
31+
@Test
32+
public void benchmarkStateConnectsToExternalStandaloneServerWhenEnvSet() throws Exception {
33+
try (StandaloneBenchmarkServer externalServer = StandaloneBenchmarkServer.start(0, 0)) {
34+
BenchmarkState state = new BenchmarkState();
35+
state.setup(
36+
String.valueOf(externalServer.getSpannerPort()),
37+
String.valueOf(externalServer.getMonitoringPort()));
38+
try {
39+
assertThat(System.getProperty("jmh.monitoring-server-port"))
40+
.isEqualTo(String.valueOf(externalServer.getMonitoringPort()));
41+
42+
ReadBenchmark benchmark = new ReadBenchmark();
43+
benchmark.readBenchmark(state, null);
44+
benchmark.queryBenchmark(state, null);
45+
46+
assertThat(
47+
externalServer.getMockSpannerService().countRequestsOfType(ExecuteSqlRequest.class))
48+
.isEqualTo(0);
49+
assertThat(externalServer.getMockSpannerService().countRequestsOfType(ReadRequest.class))
50+
.isEqualTo(0);
51+
} finally {
52+
state.tearDown();
53+
}
54+
}
55+
}
56+
57+
@Test(expected = IllegalStateException.class)
58+
public void benchmarkStateSetupThrowsWhenSpannerPortMissing() throws Exception {
59+
BenchmarkState state = new BenchmarkState();
60+
state.setup(null, null);
61+
}
62+
}

0 commit comments

Comments
 (0)