Fix ConfigNode client failover with stale candidates (#18847)

Refresh candidates before fallback selection and resume after the current endpoint.
Preserve candidates and cursor on empty refreshes.
Add routing and connection tests and re-enable node status failover IT.

Author: d-wang-commit

integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java

@@ -37,8 +37,8 @@
 import com.sun.tools.attach.VirtualMachine;
 import org.apache.tsfile.utils.Pair;
 import org.junit.After;
-import org.junit.Ignore;
 import org.junit.Rule;
+import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ErrorCollector;
 import org.junit.rules.TemporaryFolder;
@@ -77,7 +77,6 @@
 /** Process-level status persistence checks with three actual ConfigRegion voters. */
 @RunWith(IoTDBTestRunner.class)
 @Category(ClusterIT.class)
-@Ignore("Temporarily skip IoTDBNodeStatusFailoverIT")
 public class IoTDBNodeStatusFailoverIT {
   @Rule public TemporaryFolder temporary = new TemporaryFolder();
   @Rule public ErrorCollector failures = new ErrorCollector();
@@ -89,7 +88,7 @@ public void tearDown() {
     EnvFactory.getEnv().cleanClusterEnvironment();
   }
 
-  // @Test
+  @Test
   public void testGracefulLeaderShutdownPreservesStoppedAndRemoving() throws Exception {
     init(3);
     int stopped = stoppedFixture(0);
@@ -114,7 +113,7 @@ public void testGracefulLeaderShutdownPreservesStoppedAndRemoving() throws Excep
     assertStatus(1, removing, NodeStatus.Removing);
   }
 
-  // @Test
+  @Test
   public void testGracefulSeedShutdownReportsStopped() throws Exception {
     init(1);
     transfer(1);
@@ -137,7 +136,7 @@ public void testGracefulSeedShutdownReportsStopped() throws Exception {
     awaitPersisted(2, seedId, null);
   }
 
-  // @Test
+  @Test
   public void testForcedLeaderShutdownWithTwoDataNodes() throws Exception {
     init(2);
     int stopped = stoppedFixture(0);
@@ -157,7 +156,7 @@ public void testForcedLeaderShutdownWithTwoDataNodes() throws Exception {
     assertStatus(0, stopped, NodeStatus.Stopped);
   }
 
-  // @Test
+  @Test
   public void testNewLeaderChangesSurviveReturnToOriginalLeader() throws Exception {
     init(3);
     int stopped = stoppedFixture(0);
@@ -181,7 +180,7 @@ public void testNewLeaderChangesSurviveReturnToOriginalLeader() throws Exception
     assertStatus(1, removing, NodeStatus.Unknown);
   }
 
-  // @Test
+  @Test
   public void testReadOnlyReasonPersistsAcrossLeaderTransferAndClearsAfterRestart()
       throws Exception {
     init(2);
@@ -230,7 +229,7 @@ public void testReadOnlyReasonPersistsAcrossLeaderTransferAndClearsAfterRestart(
     }
   }
 
-  // @Test
+  @Test
   public void testLiveOldLeaderPartitionAndQuorumRecovery() throws Exception {
     init(3);
     int stopped = stoppedFixture(0);
@@ -299,7 +298,7 @@ private void init(int dataNodes) throws Exception {
     }
   }
 
-  // @Test
+  @Test
   public void testNoQuorumTransitionMatrix() throws Exception {
     init(2);
     int nodeId = stoppedFixture(0);
@@ -362,7 +361,7 @@ public void testNoQuorumTransitionMatrix() throws Exception {
     }
   }
 
-  // @Test
+  @Test
   public void testSnapshotAndFollowingClearLogRecoverTogether() throws Exception {
     init(3);
     int stopped = stoppedFixture(0);
@@ -458,7 +457,7 @@ private void awaitConfigNodeRunning(int index) {
             Collections.singletonList(NodeStatus.Running));
   }
 
-  // @Test
+  @Test
   public void testRealRemovalPersistsIntentAndDeletesRegistration() throws Exception {
     EnvFactory.getEnv()
         .getConfig()

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/AbstractConfigNodeClient.java

@@ -322,6 +322,11 @@ private void tryToConnect(int timeoutMs) throws TException {
       }
     }
 
+    // Refresh stale candidates before retrying after an RPC or connection failure.
+    syncLatestConfigNodeList();
+    if (configNode != null) {
+      cursor = configNodes.indexOf(configNode);
+    }
     for (int tryHostNum = 0; tryHostNum < configNodes.size(); tryHostNum++) {
       cursor = (cursor + 1) % configNodes.size();
       TEndPoint tryEndpoint = configNodes.get(cursor);
@@ -345,8 +350,11 @@ private void tryToConnect(int timeoutMs) throws TException {
   }
 
   public void syncLatestConfigNodeList() {
-    configNodes = getConfigNodeInfo().getLatestConfigNodes();
-    cursor = 0;
+    final List<TEndPoint> latestConfigNodes = getConfigNodeInfo().getLatestConfigNodes();
+    if (!latestConfigNodes.isEmpty()) {
+      configNodes = latestConfigNodes;
+      cursor = 0;
+    }
   }
 
   @Override