Skip to content

Commit 8fa65e0

Browse files
sakthivelmaniilqiu96
authored andcommitted
fix(spanner): do not cache retryable errors in AbstractLazyInitializer (#14752)
## Problem When `AbstractLazyInitializer.get()` is invoked and `initialize()` throws an exception, the failure was previously permanently cached in `this.error` with `this.initialized = true` regardless of whether the error was transient or retryable. In `MultiplexedSessionDatabaseClient.getDialect()` and `DatabaseClientImpl.getStatementFactory()`, an initial transient failure during session creation or query execution (`DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, or `UNAVAILABLE`) caused all subsequent calls to immediately rethrow the cached exception without retrying, even after multiplexed session creation succeeded on retry. ## Solution Update `AbstractLazyInitializer` to introduce a `protected boolean shouldCacheError(Exception)` method that returns `true` by default, while allowing subclasses and client components to override it to avoid caching transient or retryable errors. - In `AbstractLazyInitializer`, define `protected boolean shouldCacheError(Exception)` returning `true` by default so retryable errors can be selectively retried on subsequent `get()` calls without permanent caching. - In `MultiplexedSessionDatabaseClient` (`dialectSupplier`), override `shouldCacheError(Exception)` to not cache retryable `SpannerException` errors matching `RETRYABLE_ERROR_CODES` (`DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `UNAVAILABLE`). - In `DatabaseClientImpl`, simplify `statementFactorySupplier` to call `new StatementFactory(getDialect())` directly and override `shouldCacheError(Exception)` to return `false` (delegating error caching and retry lifecycle entirely to `getDialect()`), and remove unused `getDialectAsync()` and unused imports. - Add unit tests in `LazySpannerInitializerTest` and `DatabaseClientImplTest` covering retryable error recovery, non-retryable error caching, and concurrent retries.
1 parent 96fa0ee commit 8fa65e0

5 files changed

Lines changed: 255 additions & 19 deletions

File tree

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AbstractLazyInitializer.java‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818

1919
/**
2020
* Generic {@link AbstractLazyInitializer} for any heavy-weight object that might throw an exception
21-
* during initialization. The underlying object is initialized at most once.
21+
* during initialization. The underlying object is initialized at most once unless an exception is
22+
* thrown and {@link #shouldCacheError(Exception)} returns false.
2223
*
2324
* @param <T> Object which is to be initialized lazily
2425
*/
@@ -36,10 +37,16 @@ public T get() throws Exception {
3637
if (!initialized) {
3738
try {
3839
object = initialize();
40+
error = null;
41+
initialized = true;
3942
} catch (Exception e) {
40-
error = e;
43+
if (shouldCacheError(e)) {
44+
error = e;
45+
initialized = true;
46+
} else {
47+
throw e;
48+
}
4149
}
42-
initialized = true;
4350
}
4451
}
4552
}
@@ -49,6 +56,22 @@ public T get() throws Exception {
4956
return object;
5057
}
5158

59+
/**
60+
* Determines whether an exception thrown by {@link #initialize()} should be cached. If cached,
61+
* subsequent calls to {@link #get()} will immediately rethrow the cached exception. If not
62+
* cached, the exception is rethrown without marking this initializer as initialized, allowing
63+
* subsequent calls to retry {@link #initialize()}.
64+
*
65+
* <p>Defaults to {@code true}. Subclasses can override this method to avoid caching transient or
66+
* retryable errors.
67+
*
68+
* @param e the exception thrown during initialization
69+
* @return {@code true} if the exception should be cached, {@code false} otherwise
70+
*/
71+
protected boolean shouldCacheError(Exception e) {
72+
return true;
73+
}
74+
5275
/**
5376
* Initializes the actual object that should be returned. Is called once the first time an
5477
* instance of T is required.

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClientImpl.java‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,6 @@
3030
import io.opentelemetry.api.common.Attributes;
3131
import java.util.HashMap;
3232
import java.util.Map;
33-
import java.util.concurrent.ExecutionException;
34-
import java.util.concurrent.Future;
35-
import java.util.concurrent.TimeUnit;
36-
import java.util.concurrent.TimeoutException;
3733
import java.util.concurrent.atomic.AtomicInteger;
3834
import javax.annotation.Nullable;
3935

@@ -88,14 +84,15 @@ public Dialect getDialect() {
8884
new AbstractLazyInitializer<StatementFactory>() {
8985
@Override
9086
protected StatementFactory initialize() {
91-
try {
92-
Dialect dialect = getDialectAsync().get(30, TimeUnit.SECONDS);
93-
return new StatementFactory(dialect);
94-
} catch (ExecutionException | TimeoutException e) {
95-
throw SpannerExceptionFactory.asSpannerException(e);
96-
} catch (InterruptedException e) {
97-
throw SpannerExceptionFactory.propagateInterrupt(e);
98-
}
87+
return new StatementFactory(getDialect());
88+
}
89+
90+
@Override
91+
protected boolean shouldCacheError(Exception e) {
92+
// getDialect() already decides whether to cache or retry dialect errors.
93+
// Never cache errors here so that statementFactorySupplier always delegates
94+
// the retry/caching lifecycle entirely to getDialect().
95+
return false;
9996
}
10097
};
10198

@@ -298,10 +295,6 @@ public long executePartitionedUpdate(final Statement stmt, final UpdateOption...
298295
return multiplexedSessionDatabaseClient.executePartitionedUpdate(stmt, options);
299296
}
300297

301-
private Future<Dialect> getDialectAsync() {
302-
return multiplexedSessionDatabaseClient.getDialectAsync();
303-
}
304-
305298
boolean isValid() {
306299
return multiplexedSessionDatabaseClient.isValid();
307300
}

‎java-spanner/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MultiplexedSessionDatabaseClient.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,13 @@ private SharedChannelUsage(int numChannels) {
182182
private static final EnumSet<ErrorCode> RETRYABLE_ERROR_CODES =
183183
EnumSet.of(ErrorCode.DEADLINE_EXCEEDED, ErrorCode.RESOURCE_EXHAUSTED, ErrorCode.UNAVAILABLE);
184184

185+
static boolean shouldCacheDialectError(Exception e) {
186+
if (e instanceof SpannerException) {
187+
return !RETRYABLE_ERROR_CODES.contains(((SpannerException) e).getErrorCode());
188+
}
189+
return true;
190+
}
191+
185192
private final BitSet channelUsage;
186193

187194
private final int numChannels;
@@ -539,6 +546,11 @@ protected Dialect initialize() {
539546
// This should not really happen, but it is the safest fallback value.
540547
return Dialect.GOOGLE_STANDARD_SQL;
541548
}
549+
550+
@Override
551+
protected boolean shouldCacheError(Exception e) {
552+
return shouldCacheDialectError(e);
553+
}
542554
};
543555

544556
@Override

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseClientImplTest.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import static org.junit.Assert.assertFalse;
3030
import static org.junit.Assert.assertNotNull;
3131
import static org.junit.Assert.assertNull;
32+
import static org.junit.Assert.assertSame;
3233
import static org.junit.Assert.assertThrows;
3334
import static org.junit.Assert.assertTrue;
3435

@@ -115,6 +116,7 @@
115116
import java.util.concurrent.Executors;
116117
import java.util.concurrent.TimeUnit;
117118
import java.util.concurrent.atomic.AtomicInteger;
119+
import java.util.concurrent.atomic.AtomicReference;
118120
import java.util.function.Function;
119121
import java.util.logging.Level;
120122
import java.util.logging.Logger;
@@ -3145,6 +3147,122 @@ public void testGetDialectDefaultPreloaded_FailsDirectlyIfDatabaseNotFound() {
31453147
}
31463148
}
31473149

3150+
@Test
3151+
public void testGetDialect_RecoversAfterTransientCreateSessionFailure() throws Exception {
3152+
mockSpanner.putStatementResult(StatementResult.detectDialectResult(Dialect.POSTGRESQL));
3153+
mockSpanner.setCreateSessionExecutionTime(
3154+
SimulatedExecutionTime.ofException(
3155+
Status.DEADLINE_EXCEEDED
3156+
.withDescription("Simulated transient outage on createSession call #1")
3157+
.asRuntimeException()));
3158+
mockSpanner.freeze();
3159+
try {
3160+
DatabaseClient client =
3161+
spanner.getDatabaseClient(DatabaseId.of(TEST_PROJECT, TEST_INSTANCE, TEST_DATABASE));
3162+
AtomicReference<Throwable> firstCallError = new AtomicReference<>();
3163+
Thread firstCaller =
3164+
new Thread(
3165+
() -> {
3166+
try {
3167+
client.getDialect();
3168+
} catch (Throwable t) {
3169+
firstCallError.set(t);
3170+
}
3171+
});
3172+
firstCaller.start();
3173+
// Wait (up to 10s) until firstCaller is blocked waiting on the initial multiplexed session.
3174+
Stopwatch watch = Stopwatch.createStarted();
3175+
while (firstCaller.getState() != Thread.State.WAITING
3176+
&& firstCaller.getState() != Thread.State.TIMED_WAITING
3177+
&& firstCaller.getState() != Thread.State.TERMINATED
3178+
&& watch.elapsed(TimeUnit.SECONDS) < 10) {
3179+
Thread.yield();
3180+
}
3181+
mockSpanner.unfreeze();
3182+
firstCaller.join(10_000L);
3183+
assertFalse(firstCaller.isAlive());
3184+
3185+
assertThat(firstCallError.get()).isInstanceOf(SpannerException.class);
3186+
SpannerException firstException = (SpannerException) firstCallError.get();
3187+
assertEquals(ErrorCode.DEADLINE_EXCEEDED, firstException.getErrorCode());
3188+
3189+
// Wait for the retry CreateSession request to reach the server so multiplexedSessionReference
3190+
// has been updated to the new session future.
3191+
mockSpanner.waitForRequestsToContain(
3192+
message -> message instanceof CreateSessionRequest, 10_000L);
3193+
try (ResultSet rs = client.singleUse().executeQuery(SELECT1)) {
3194+
assertTrue(rs.next());
3195+
}
3196+
3197+
assertEquals(Dialect.POSTGRESQL, client.getDialect());
3198+
assertEquals(Dialect.POSTGRESQL, client.getDialect());
3199+
assertEquals(
3200+
Statement.newBuilder("select id from test where b=$1").bind("p1").to(true).build(),
3201+
client
3202+
.getStatementFactory()
3203+
.withUnnamedParameters("select id from test where b=?", true));
3204+
} finally {
3205+
mockSpanner.unfreeze();
3206+
mockSpanner.putStatementResult(
3207+
StatementResult.detectDialectResult(Dialect.GOOGLE_STANDARD_SQL));
3208+
}
3209+
}
3210+
3211+
@Test
3212+
public void testGetDialectAndStatementFactory_RecoverAfterTransientQueryFailure() {
3213+
mockSpanner.putStatementResult(
3214+
StatementResult.exception(
3215+
MultiplexedSessionDatabaseClient.DETERMINE_DIALECT_STATEMENT,
3216+
Status.DEADLINE_EXCEEDED
3217+
.withDescription("Transient dialect query timeout")
3218+
.asRuntimeException()));
3219+
try {
3220+
DatabaseClient client =
3221+
spanner.getDatabaseClient(DatabaseId.of(TEST_PROJECT, TEST_INSTANCE, TEST_DATABASE));
3222+
3223+
SpannerException dialectException = assertThrows(SpannerException.class, client::getDialect);
3224+
assertEquals(ErrorCode.DEADLINE_EXCEEDED, dialectException.getErrorCode());
3225+
3226+
SpannerException factoryException =
3227+
assertThrows(SpannerException.class, client::getStatementFactory);
3228+
assertEquals(ErrorCode.DEADLINE_EXCEEDED, factoryException.getErrorCode());
3229+
3230+
mockSpanner.putStatementResult(StatementResult.detectDialectResult(Dialect.POSTGRESQL));
3231+
3232+
assertEquals(Dialect.POSTGRESQL, client.getDialect());
3233+
assertEquals(
3234+
Statement.newBuilder("select id from test where b=$1").bind("p1").to(true).build(),
3235+
client
3236+
.getStatementFactory()
3237+
.withUnnamedParameters("select id from test where b=?", true));
3238+
} finally {
3239+
mockSpanner.putStatementResult(
3240+
StatementResult.detectDialectResult(Dialect.GOOGLE_STANDARD_SQL));
3241+
}
3242+
}
3243+
3244+
@Test
3245+
public void testGetDialect_CachesNonRetryableQueryFailure() {
3246+
mockSpanner.putStatementResult(
3247+
StatementResult.exception(
3248+
MultiplexedSessionDatabaseClient.DETERMINE_DIALECT_STATEMENT,
3249+
Status.PERMISSION_DENIED
3250+
.withDescription("Permission denied on information_schema")
3251+
.asRuntimeException()));
3252+
try {
3253+
DatabaseClient client =
3254+
spanner.getDatabaseClient(DatabaseId.of(TEST_PROJECT, TEST_INSTANCE, "non-retryable-db"));
3255+
3256+
SpannerException e1 = assertThrows(SpannerException.class, client::getDialect);
3257+
assertEquals(ErrorCode.PERMISSION_DENIED, e1.getErrorCode());
3258+
SpannerException e2 = assertThrows(SpannerException.class, client::getDialect);
3259+
assertSame(e1, e2);
3260+
} finally {
3261+
mockSpanner.putStatementResult(
3262+
StatementResult.detectDialectResult(Dialect.GOOGLE_STANDARD_SQL));
3263+
}
3264+
}
3265+
31483266
@Test
31493267
public void testUntypedNullParameters() {
31503268
Statement statement =

‎java-spanner/google-cloud-spanner/src/test/java/com/google/cloud/spanner/LazySpannerInitializerTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,4 +100,94 @@ public Spanner initialize() {
100100
assertThat(count.get()).isEqualTo(1);
101101
executor.shutdown();
102102
}
103+
104+
@Test
105+
public void testGet_shouldRetryOnRetryableSpannerException() throws Exception {
106+
final AtomicInteger attempts = new AtomicInteger();
107+
final Spanner expectedSpanner = mock(Spanner.class);
108+
final LazySpannerInitializer initializer =
109+
new LazySpannerInitializer() {
110+
@Override
111+
public Spanner initialize() {
112+
int attempt = attempts.incrementAndGet();
113+
if (attempt == 1) {
114+
throw SpannerExceptionFactory.newSpannerException(
115+
ErrorCode.UNAVAILABLE, "Transient unavailable");
116+
}
117+
if (attempt == 2) {
118+
throw SpannerExceptionFactory.newSpannerException(
119+
ErrorCode.DEADLINE_EXCEEDED, "Transient deadline exceeded");
120+
}
121+
return expectedSpanner;
122+
}
123+
124+
@Override
125+
protected boolean shouldCacheError(Exception e) {
126+
return MultiplexedSessionDatabaseClient.shouldCacheDialectError(e);
127+
}
128+
};
129+
SpannerException e1 = assertThrows(SpannerException.class, initializer::get);
130+
assertThat(e1.getErrorCode()).isEqualTo(ErrorCode.UNAVAILABLE);
131+
SpannerException e2 = assertThrows(SpannerException.class, initializer::get);
132+
assertThat(e2.getErrorCode()).isEqualTo(ErrorCode.DEADLINE_EXCEEDED);
133+
134+
Spanner s1 = initializer.get();
135+
Spanner s2 = initializer.get();
136+
assertThat(s1).isSameInstanceAs(expectedSpanner);
137+
assertThat(s2).isSameInstanceAs(expectedSpanner);
138+
assertThat(attempts.get()).isEqualTo(3);
139+
}
140+
141+
@Test
142+
public void testGet_concurrentRetryAfterRetryableFailure() throws InterruptedException {
143+
final AtomicInteger count = new AtomicInteger();
144+
final Spanner expectedSpanner = mock(Spanner.class);
145+
final LazySpannerInitializer initializer =
146+
new LazySpannerInitializer() {
147+
@Override
148+
public Spanner initialize() {
149+
if (count.incrementAndGet() == 1) {
150+
throw SpannerExceptionFactory.newSpannerException(
151+
ErrorCode.UNAVAILABLE, "Transient error");
152+
}
153+
return expectedSpanner;
154+
}
155+
156+
@Override
157+
protected boolean shouldCacheError(Exception e) {
158+
return MultiplexedSessionDatabaseClient.shouldCacheDialectError(e);
159+
}
160+
};
161+
final int threads = 16;
162+
final CountDownLatch latch = new CountDownLatch(threads);
163+
ListeningExecutorService executor =
164+
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threads));
165+
List<ListenableFuture<Spanner>> futures = new ArrayList<>(threads);
166+
for (int i = 0; i < threads; i++) {
167+
futures.add(
168+
executor.submit(
169+
() -> {
170+
latch.countDown();
171+
latch.await(10L, TimeUnit.SECONDS);
172+
return initializer.get();
173+
}));
174+
}
175+
int failureCount = 0;
176+
int successCount = 0;
177+
for (ListenableFuture<Spanner> future : futures) {
178+
try {
179+
assertThat(future.get()).isSameInstanceAs(expectedSpanner);
180+
successCount++;
181+
} catch (ExecutionException e) {
182+
assertThat(e.getCause()).isInstanceOf(SpannerException.class);
183+
assertThat(((SpannerException) e.getCause()).getErrorCode())
184+
.isEqualTo(ErrorCode.UNAVAILABLE);
185+
failureCount++;
186+
}
187+
}
188+
assertThat(failureCount).isEqualTo(1);
189+
assertThat(successCount).isEqualTo(threads - 1);
190+
assertThat(count.get()).isEqualTo(2);
191+
executor.shutdown();
192+
}
103193
}

0 commit comments

Comments
 (0)