Skip to content

Commit ce976d2

Browse files
committed
feat(spanner): switch to per-channel EEF fallback and add GetSession probe
1 parent 945a856 commit ce976d2

4 files changed

Lines changed: 173 additions & 22 deletions

File tree

‎grpc-gcp-java/src/main/java/com/google/cloud/grpc/fallback/GcpFallbackChannel.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ private void probePrimary() {
303303
&& durationSatisfied) {
304304
fallbackState.getInFallbackMode().set(false);
305305
localInFallbackMode.set(false);
306+
logger.info("[kinsaurralde] Primary channel recovered to DirectPath.");
306307
localProbeSuccesses.set(0);
307308
localFirstPrimaryProbeSuccessNanos.set(0);
308309
}

‎grpc-gcp-java/src/main/java/com/google/cloud/grpc/fallback/GcpFallbackState.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.concurrent.TimeUnit;
2424
import java.util.concurrent.atomic.AtomicBoolean;
2525
import java.util.concurrent.atomic.AtomicLong;
26+
import java.util.logging.Logger;
2627

2728
/**
2829
* Shared thread-safe state container for coordinated pool-wide failover, recovery, and background
@@ -32,6 +33,7 @@
3233
* consolidating probing and error evaluation threads across the entire channel pool.
3334
*/
3435
public class GcpFallbackState {
36+
private static final Logger logger = Logger.getLogger(GcpFallbackState.class.getName());
3537
private final AtomicLong primarySuccesses = new AtomicLong(0);
3638
private final AtomicLong primaryFailures = new AtomicLong(0);
3739
private final AtomicLong fallbackSuccesses = new AtomicLong(0);
@@ -154,6 +156,7 @@ public void checkErrorRates(
154156
if (!inFallbackMode.get() && options.isEnableFallback()) {
155157
if (failures >= options.getMinFailedCalls() && errRate >= options.getErrorRateThreshold()) {
156158
inFallbackMode.set(true);
159+
logger.info("[kinsaurralde] Primary error rate threshold crossed. Switching pool to fallback mode.");
157160
if (openTelemetry != null && openTelemetry.getModule() != null) {
158161
openTelemetry
159162
.getModule()

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 165 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,9 @@ public class GapicSpannerRpc implements SpannerRpc {
266266

267267
private static final String API_FILE = "grpc-gcp-apiconfig.json";
268268

269+
private final com.google.cloud.grpc.fallback.GcpFallbackState sharedFallbackState =
270+
new com.google.cloud.grpc.fallback.GcpFallbackState();
271+
269272
private final RequestIdCreator requestIdCreator = new RequestIdCreatorImpl();
270273
private boolean rpcIsClosed;
271274
private final SpannerStub spannerStub;
@@ -312,6 +315,9 @@ public class GapicSpannerRpc implements SpannerRpc {
312315

313316
private final GrpcCallContext baseGrpcCallContext;
314317

318+
// Inside GapicSpannerRpc class fields
319+
private final GetSessionProbe getSessionProbe = new GetSessionProbe();
320+
315321
public static GapicSpannerRpc create(SpannerOptions options) {
316322
return new GapicSpannerRpc(options);
317323
}
@@ -321,6 +327,8 @@ public GapicSpannerRpc(final SpannerOptions options) {
321327
}
322328

323329
GapicSpannerRpc(final SpannerOptions options, boolean initializeStubs) {
330+
System.out.println(
331+
"[kinsaurralde] #################2 Using Local Cloud Spanner Client #################^2");
324332
this.projectId = options.getProjectId();
325333
String projectNameStr = PROJECT_NAME_TEMPLATE.instantiate("project", this.projectId);
326334
try {
@@ -588,15 +596,26 @@ public <RequestT, ResponseT> UnaryCallable<RequestT, ResponseT> createUnaryCalla
588596
}
589597
}
590598

599+
@VisibleForTesting
600+
com.google.cloud.grpc.fallback.GcpFallbackState getSharedFallbackState() {
601+
return sharedFallbackState;
602+
}
603+
591604
@VisibleForTesting
592605
GcpFallbackChannelOptions createFallbackChannelOptions(
593606
GcpFallbackOpenTelemetry fallbackTelemetry, int minFailedCalls) {
594607
return GcpFallbackChannelOptions.newBuilder()
608+
.setSharedState(sharedFallbackState)
595609
.setPrimaryChannelName("directpath")
596610
.setFallbackChannelName("cloudpath")
597611
.setMinFailedCalls(minFailedCalls)
598612
.setPeriod(Duration.ofMinutes(3))
599613
.setGcpFallbackOpenTelemetry(fallbackTelemetry)
614+
.setPrimaryProbingFunction(getSessionProbe)
615+
.setPrimaryProbingInterval(Duration.ofSeconds(10))
616+
.setMinPrimaryProbeSuccessCount(50)
617+
.setMinPrimaryProbeSuccessDuration(Duration.ofMinutes(10))
618+
.setEnableRecovery(true)
600619
.build();
601620
}
602621

@@ -694,33 +713,30 @@ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
694713
builder = existingConfigurator.apply(builder);
695714
}
696715

697-
ManagedChannelBuilder<?> primaryBuilder = builder;
698-
ManagedChannelBuilder<?> fallbackBuilder = cloudPathBuilder;
716+
GcpFallbackOpenTelemetry fallbackTelemetry =
717+
GcpFallbackOpenTelemetry.newBuilder()
718+
.withSdk(getFallbackOpenTelemetry(options))
719+
.disableAllMetrics()
720+
.enableMetrics(Arrays.asList("fallback_count", "call_status"))
721+
.build();
722+
723+
FallbackChannelBuilder fallbackDelegate =
724+
new FallbackChannelBuilder(
725+
builder, cloudPathBuilder, createFallbackChannelOptions(fallbackTelemetry, 1));
726+
699727
if (options.isGrpcGcpExtensionEnabled()) {
700728
String jsonApiConfig = parseGrpcGcpApiConfig();
701729
GcpManagedChannelOptions gcpOptions = grpcGcpOptionsWithMetricsAndDcp(options);
702730
if (gcpOptions == null) {
703731
gcpOptions = GcpManagedChannelOptions.newBuilder().build();
704732
}
705-
primaryBuilder =
706-
GcpManagedChannelBuilder.forDelegateBuilder(builder)
707-
.withApiConfigJsonString(jsonApiConfig)
708-
.withOptions(gcpOptions);
709-
fallbackBuilder =
710-
GcpManagedChannelBuilder.forDelegateBuilder(cloudPathBuilder)
711-
.withApiConfigJsonString(jsonApiConfig)
712-
.withOptions(gcpOptions);
713-
}
714733

715-
GcpFallbackOpenTelemetry fallbackTelemetry =
716-
GcpFallbackOpenTelemetry.newBuilder()
717-
.withSdk(getFallbackOpenTelemetry(options))
718-
.disableAllMetrics()
719-
.enableMetrics(Arrays.asList("fallback_count", "call_status"))
720-
.build();
734+
return GcpManagedChannelBuilder.forDelegateBuilder(fallbackDelegate)
735+
.withApiConfigJsonString(jsonApiConfig)
736+
.withOptions(gcpOptions);
737+
}
721738

722-
return new FallbackChannelBuilder(
723-
primaryBuilder, fallbackBuilder, createFallbackChannelOptions(fallbackTelemetry, 1));
739+
return fallbackDelegate;
724740
});
725741
}
726742

@@ -751,6 +767,7 @@ private InstantiatingGrpcChannelProvider.Builder createBaseChannelProviderBuilde
751767
// commit GRPC calls succeed
752768
.setKeepAliveTimeDuration(options.getGrpcKeepAliveTime())
753769
.setKeepAliveTimeoutDuration(options.getGrpcKeepAliveTimeout())
770+
.setKeepAliveWithoutCalls(true)
754771

755772
// Then check if SpannerOptions provides an InterceptorProvider. Create a default
756773
// SpannerInterceptorProvider if none is provided
@@ -1969,7 +1986,15 @@ public Session createSession(
19691986
CreateSessionRequest request = requestBuilder.build();
19701987
GrpcCallContext context =
19711988
newCallContext(options, databaseName, request, SpannerGrpc.getCreateSessionMethod(), true);
1972-
return get(spannerStub.createSessionCallable().futureCall(request, context));
1989+
// Execute the call
1990+
Session session = get(spannerStub.createSessionCallable().futureCall(request, context));
1991+
1992+
// Capture the multiplexed session name for the prober
1993+
if (isMultiplexed && session != null) {
1994+
getSessionProbe.setSessionName(session.getName());
1995+
}
1996+
1997+
return session;
19731998
}
19741999

19752000
@Override
@@ -2446,6 +2471,7 @@ public void shutdown() {
24462471
this.instanceAdminStub.close();
24472472
this.databaseAdminStub.close();
24482473
this.spannerWatchdog.shutdown();
2474+
this.sharedFallbackState.shutdown();
24492475

24502476
try {
24512477
this.spannerStub.awaitTermination(10L, TimeUnit.SECONDS);
@@ -2467,6 +2493,7 @@ public void shutdownNow() {
24672493
this.instanceAdminStub.close();
24682494
this.databaseAdminStub.close();
24692495
this.spannerWatchdog.shutdown();
2496+
this.sharedFallbackState.shutdownNow();
24702497

24712498
this.spannerStub.shutdownNow();
24722499
this.partitionedDmlStub.shutdownNow();
@@ -2578,6 +2605,124 @@ private static Duration systemProperty(String name, int defaultValue) {
25782605
return Duration.ofSeconds(stringValue.isEmpty() ? defaultValue : Integer.parseInt(stringValue));
25792606
}
25802607

2608+
private class GetSessionProbe
2609+
implements com.google.common.base.Function<io.grpc.Channel, String> {
2610+
private final java.util.concurrent.atomic.AtomicReference<String> sessionName =
2611+
new java.util.concurrent.atomic.AtomicReference<>(null);
2612+
2613+
public void setSessionName(String name) {
2614+
System.out.println(
2615+
"[kinsaurralde] @@@@@@@@@@@@@@ Setting session name for GetSession Probe @@@@@@@@@@@@@@");
2616+
this.sessionName.set(name);
2617+
}
2618+
2619+
@Override
2620+
public String apply(io.grpc.Channel channel) {
2621+
String name = sessionName.get();
2622+
if (name == null) {
2623+
return "WAITING_FOR_REAL_SESSION";
2624+
}
2625+
2626+
try {
2627+
com.google.spanner.v1.GetSessionRequest request =
2628+
com.google.spanner.v1.GetSessionRequest.newBuilder().setName(name).build();
2629+
2630+
java.util.Map<String, java.util.List<String>> extraHeaders =
2631+
metadataProvider.newExtraHeaders(name, projectName);
2632+
2633+
io.grpc.Metadata metadata = new io.grpc.Metadata();
2634+
for (java.util.Map.Entry<String, java.util.List<String>> entry : extraHeaders.entrySet()) {
2635+
io.grpc.Metadata.Key<String> key =
2636+
io.grpc.Metadata.Key.of(entry.getKey(), io.grpc.Metadata.ASCII_STRING_MARSHALLER);
2637+
if (entry.getValue() != null) {
2638+
for (String value : entry.getValue()) {
2639+
metadata.put(key, value);
2640+
}
2641+
}
2642+
}
2643+
2644+
final java.util.concurrent.atomic.AtomicReference<io.grpc.Attributes> capturedAttributes =
2645+
new java.util.concurrent.atomic.AtomicReference<>();
2646+
2647+
io.grpc.ClientInterceptor attributeInterceptor =
2648+
new io.grpc.ClientInterceptor() {
2649+
@Override
2650+
public <ReqT, RespT> io.grpc.ClientCall<ReqT, RespT> interceptCall(
2651+
io.grpc.MethodDescriptor<ReqT, RespT> method,
2652+
io.grpc.CallOptions callOptions,
2653+
io.grpc.Channel next) {
2654+
io.grpc.ClientCall<ReqT, RespT> call = next.newCall(method, callOptions);
2655+
return new io.grpc.ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
2656+
call) {
2657+
@Override
2658+
public void start(Listener<RespT> responseListener, io.grpc.Metadata headers) {
2659+
super.start(
2660+
new io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener<
2661+
RespT>(responseListener) {
2662+
@Override
2663+
public void onHeaders(io.grpc.Metadata headers) {
2664+
capturedAttributes.set(call.getAttributes());
2665+
super.onHeaders(headers);
2666+
}
2667+
},
2668+
headers);
2669+
}
2670+
};
2671+
}
2672+
};
2673+
2674+
// Direct synchronous probe RPC executed on the specific underlying channel.
2675+
com.google.spanner.v1.SpannerGrpc.SpannerBlockingStub stub =
2676+
com.google.spanner.v1.SpannerGrpc.newBlockingStub(channel)
2677+
.withInterceptors(
2678+
io.grpc.stub.MetadataUtils.newAttachHeadersInterceptor(metadata),
2679+
attributeInterceptor)
2680+
.withDeadlineAfter(10, java.util.concurrent.TimeUnit.SECONDS);
2681+
2682+
if (callCredentialsProvider != null) {
2683+
io.grpc.CallCredentials creds = callCredentialsProvider.getCallCredentials();
2684+
if (creds != null) {
2685+
stub = stub.withCallCredentials(creds);
2686+
}
2687+
}
2688+
2689+
stub.getSession(request);
2690+
2691+
io.grpc.Attributes attrs = capturedAttributes.get();
2692+
if (attrs != null) {
2693+
java.net.SocketAddress remoteAddr = attrs.get(io.grpc.Grpc.TRANSPORT_ATTR_REMOTE_ADDR);
2694+
if (!isDirectPathTransport(remoteAddr, attrs)) {
2695+
System.out.println(
2696+
"[kinsaurralde] Probe succeeded but NOT using DirectPath transport (remoteAddr: "
2697+
+ remoteAddr
2698+
+ ")");
2699+
return "NOT_DIRECTPATH";
2700+
}
2701+
}
2702+
2703+
System.out.println(
2704+
"[kinsaurralde] GetSession Probe OK for single channel (DirectPath verified)!");
2705+
return "OK";
2706+
} catch (io.grpc.StatusRuntimeException e) {
2707+
System.out.println("[kinsaurralde] Probe FAILED with " + e.getStatus().getCode().name());
2708+
return e.getStatus().getCode().name();
2709+
} catch (Exception e) {
2710+
return "ERROR";
2711+
}
2712+
}
2713+
2714+
private boolean isDirectPathTransport(
2715+
java.net.SocketAddress remoteAddr, io.grpc.Attributes attrs) {
2716+
// DirectPath (both IPv4 and IPv6) uses ALTS security handshaking.
2717+
// CloudPath (GFE) uses standard TLS/SSL certificates.
2718+
Object sslSession = attrs.get(io.grpc.Grpc.TRANSPORT_ATTR_SSL_SESSION);
2719+
if (sslSession != null) {
2720+
return false; // Standard TLS session indicates GFE (CloudPath)
2721+
}
2722+
return true; // ALTS transport indicates DirectPath (IPv4 or IPv6)
2723+
}
2724+
}
2725+
25812726
// Wrapper class to build the GcpFallbackChannel using GAX's configuration
25822727
private static class FallbackChannelBuilder
25832728
extends ForwardingChannelBuilder2<FallbackChannelBuilder> {

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1520,8 +1520,8 @@ public void testDirectPathFallbackCreatesOneGrpcGcpLayerPerPath() {
15201520
GrpcGcpObjectCounts before = countGrpcGcpObjectsFromChannelz();
15211521
rpc = new GapicSpannerRpc(options);
15221522
GrpcGcpObjectCounts counts = countGrpcGcpObjectsFromChannelz().minus(before);
1523-
assertEquals(counts.debugString(), 6, counts.gcpManagedChannels);
1524-
assertEquals(counts.debugString(), 48, counts.channelRefs);
1523+
assertEquals(counts.debugString(), 3, counts.gcpManagedChannels);
1524+
assertEquals(counts.debugString(), 24, counts.channelRefs);
15251525
} finally {
15261526
if (rpc != null) {
15271527
rpc.shutdown();
@@ -1651,6 +1651,7 @@ GcpFallbackChannelOptions createFallbackChannelOptions(
16511651
GcpFallbackOpenTelemetry fallbackTelemetry, int minFailedCalls) {
16521652
// Override default 1-minute period to 10ms for instant testing
16531653
return GcpFallbackChannelOptions.newBuilder()
1654+
.setSharedState(getSharedFallbackState())
16541655
.setPrimaryChannelName("directpath")
16551656
.setFallbackChannelName("cloudpath")
16561657
.setMinFailedCalls(10)
@@ -1730,6 +1731,7 @@ GcpFallbackChannelOptions createFallbackChannelOptions(
17301731
GcpFallbackOpenTelemetry fallbackTelemetry, int minFailedCalls) {
17311732
// Override default 1-minute period to 10ms for instant testing
17321733
return GcpFallbackChannelOptions.newBuilder()
1734+
.setSharedState(getSharedFallbackState())
17331735
.setPrimaryChannelName("directpath")
17341736
.setFallbackChannelName("cloudpath")
17351737
.setMinFailedCalls(1)

0 commit comments

Comments
 (0)