Skip to content

Commit 945a856

Browse files
committed
fixes
1 parent 5321aa2 commit 945a856

3 files changed

Lines changed: 94 additions & 6 deletions

File tree

‎grpc-gcp-java/src/main/java/com/google/cloud/grpc/fallback/GcpFallbackChannel.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ public class GcpFallbackChannel extends ManagedChannel {
5959
private final AtomicLong localFirstPrimaryProbeSuccessNanos = new AtomicLong(0);
6060

6161
private final ScheduledExecutorService execService;
62-
private ScheduledFuture<?> primaryProbeFuture = null;
63-
private ScheduledFuture<?> fallbackProbeFuture = null;
62+
private volatile ScheduledFuture<?> primaryProbeFuture = null;
63+
private volatile ScheduledFuture<?> fallbackProbeFuture = null;
6464

6565
public GcpFallbackChannel(
6666
GcpFallbackChannelOptions options,
@@ -187,6 +187,11 @@ public boolean isInFallbackMode() {
187187
localProbeSuccesses.set(0);
188188
localFirstPrimaryProbeSuccessNanos.set(0);
189189
}
190+
} else if (!options.isEnablePerChannelRecovery()) {
191+
if (localInFallbackMode.compareAndSet(true, false)) {
192+
localProbeSuccesses.set(0);
193+
localFirstPrimaryProbeSuccessNanos.set(0);
194+
}
190195
}
191196
if (options.isEnablePerChannelRecovery()) {
192197
return (localInFallbackMode.get() && fallbackChannel != null) || primaryChannel == null;
@@ -260,6 +265,11 @@ private void probePrimary() {
260265
localProbeSuccesses.set(0);
261266
localFirstPrimaryProbeSuccessNanos.set(0);
262267
}
268+
} else if (!options.isEnablePerChannelRecovery()) {
269+
if (localInFallbackMode.compareAndSet(true, false)) {
270+
localProbeSuccesses.set(0);
271+
localFirstPrimaryProbeSuccessNanos.set(0);
272+
}
263273
}
264274
boolean inFallback =
265275
options.isEnablePerChannelRecovery()
@@ -275,15 +285,16 @@ private void probePrimary() {
275285
result = options.getPrimaryProbingFunction().apply(primaryDelegateChannel);
276286
}
277287
if ("OK".equals(result)) {
278-
localFirstPrimaryProbeSuccessNanos.compareAndSet(0, System.nanoTime());
279-
long firstSuccessNanos = localFirstPrimaryProbeSuccessNanos.get();
288+
long nowNanos = System.nanoTime();
289+
long firstSuccessNanos =
290+
localFirstPrimaryProbeSuccessNanos.updateAndGet(prev -> prev == 0 ? nowNanos : prev);
280291
long primaryProbeSuccessCount = localProbeSuccesses.incrementAndGet();
281292

282293
boolean durationSatisfied = true;
283294
if (options.getMinPrimaryProbeSuccessDuration() != null
284295
&& !options.getMinPrimaryProbeSuccessDuration().isZero()
285296
&& !options.getMinPrimaryProbeSuccessDuration().isNegative()) {
286-
long elapsedNanos = System.nanoTime() - firstSuccessNanos;
297+
long elapsedNanos = nowNanos - firstSuccessNanos;
287298
durationSatisfied = elapsedNanos >= options.getMinPrimaryProbeSuccessDuration().toNanos();
288299
}
289300

‎grpc-gcp-java/src/main/java/com/google/cloud/grpc/fallback/GcpFallbackState.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class GcpFallbackState {
4141

4242
private ScheduledExecutorService execService = null;
4343
private boolean ownsExecutor = false;
44-
private ScheduledFuture<?> scheduledEvaluationFuture = null;
44+
private volatile ScheduledFuture<?> scheduledEvaluationFuture = null;
4545

4646
public AtomicLong getPrimarySuccesses() {
4747
return primarySuccesses;

‎grpc-gcp-java/src/test/java/com/google/cloud/grpc/fallback/GcpFallbackChannelTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
import static org.mockito.ArgumentMatchers.anyLong;
4242
import static org.mockito.ArgumentMatchers.eq;
4343
import static org.mockito.ArgumentMatchers.isNull;
44+
import static org.mockito.Mockito.atLeast;
4445
import static org.mockito.Mockito.atLeastOnce;
4546
import static org.mockito.Mockito.clearInvocations;
4647
import static org.mockito.Mockito.mock;
@@ -1779,4 +1780,80 @@ public void testRecoveryDisabled_probingSucceedsButChannelRemainsInFallback() {
17791780
sharedState.shutdown();
17801781
}
17811782
}
1783+
1784+
@Test
1785+
public void testPoolLevelRecovery_multipleFailoverCyclesResetProbeStatistics() {
1786+
GcpFallbackState sharedState = new GcpFallbackState();
1787+
sharedState.getInFallbackMode().set(true); // Cycle 1: Fallback active
1788+
1789+
GcpFallbackChannelOptions options1 =
1790+
getDefaultOptionsBuilder()
1791+
.setSharedState(sharedState)
1792+
.setEnableRecovery(true)
1793+
.setEnablePerChannelRecovery(false)
1794+
.setPrimaryProbingFunction(channel -> "OK")
1795+
.setMinPrimaryProbeSuccessCount(2)
1796+
.setMinPrimaryProbeSuccessDuration(Duration.ZERO)
1797+
.build();
1798+
1799+
GcpFallbackChannelOptions options2 =
1800+
getDefaultOptionsBuilder()
1801+
.setSharedState(sharedState)
1802+
.setEnableRecovery(true)
1803+
.setEnablePerChannelRecovery(false)
1804+
.setPrimaryProbingFunction(channel -> "OK")
1805+
.setMinPrimaryProbeSuccessCount(2)
1806+
.setMinPrimaryProbeSuccessDuration(Duration.ZERO)
1807+
.build();
1808+
1809+
ScheduledExecutorService mockExec = mock(ScheduledExecutorService.class);
1810+
ArgumentCaptor<Runnable> taskCaptor = ArgumentCaptor.forClass(Runnable.class);
1811+
1812+
GcpFallbackChannel channel1 =
1813+
new GcpFallbackChannel(options1, mockPrimaryBuilder, mockFallbackBuilder, mockExec);
1814+
GcpFallbackChannel channel2 =
1815+
new GcpFallbackChannel(options2, mockPrimaryBuilder, mockFallbackBuilder, mockExec);
1816+
1817+
try {
1818+
verify(mockExec, atLeast(2))
1819+
.scheduleAtFixedRate(
1820+
taskCaptor.capture(),
1821+
eq(options1.getPrimaryProbingInterval().toMillis()),
1822+
eq(options1.getPrimaryProbingInterval().toMillis()),
1823+
eq(MILLISECONDS));
1824+
Runnable probeTask1 = taskCaptor.getAllValues().get(0);
1825+
Runnable probeTask2 = taskCaptor.getAllValues().get(1);
1826+
1827+
// Both channels enter fallback
1828+
assertTrue(channel1.isInFallbackMode());
1829+
assertTrue(channel2.isInFallbackMode());
1830+
1831+
// Channel 1 probes twice -> recovers pool
1832+
probeTask1.run();
1833+
probeTask1.run();
1834+
assertFalse(channel1.isInFallbackMode());
1835+
assertFalse(channel2.isInFallbackMode());
1836+
1837+
// Now Cycle 2: Incident occurs again, pool enters fallback
1838+
sharedState.getInFallbackMode().set(true);
1839+
assertTrue(channel2.isInFallbackMode());
1840+
1841+
// Channel 2's localProbeSuccesses must be reset to 0 in new cycle
1842+
assertEquals(0, channel2.getLocalProbeSuccesses().get());
1843+
1844+
// Probe once: count is 1 (< 2 required), must still be in fallback
1845+
probeTask2.run();
1846+
assertEquals(1, channel2.getLocalProbeSuccesses().get());
1847+
assertTrue(channel2.isInFallbackMode());
1848+
1849+
// Probe second time: count is 2 (>= 2 required) -> recovers!
1850+
probeTask2.run();
1851+
assertFalse(channel2.isInFallbackMode());
1852+
assertFalse(sharedState.getInFallbackMode().get());
1853+
} finally {
1854+
channel1.shutdownNow();
1855+
channel2.shutdownNow();
1856+
sharedState.shutdown();
1857+
}
1858+
}
17821859
}

0 commit comments

Comments
 (0)