Skip to content

Commit d461ac7

Browse files
authored
fix(db): request exit where Netty, Jetty or futures swallow TronError (#7063)
* fix(db): request exit where netty, jetty or futures swallow TronError DB_FLUSH in SnapshotManager.flush and EVENT_SUBSCRIBE_ERROR in Manager.blockTrigger relied on the TronError reaching the default uncaught handler or an ExecutorServiceManager wrapper. On the P2P broadcast-block path (netty), the HTTP broadcast servlets (Jetty) and the solidity block processing thread (plain executor future) the error was caught first, so the node kept running with flushing disabled or with lost events. Request the exit at these two throw sites through ExitManager.requestExit. A TronError raised while opening a checkpoint database during flush is now reported as DB_FLUSH with the original error as its cause, sets hitDown and requests the exit instead of being retried on later flushes. logAndExit marks a TronError once its exit thread has started and skips it afterwards, so the same error reaching an executor wrapper or the uncaught handler is not logged or exited twice. Startup-only sites are unchanged. * test(common): cover logAndExit skipping an error already exiting
1 parent 409a33f commit d461ac7

8 files changed

Lines changed: 107 additions & 11 deletions

File tree

‎chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.springframework.beans.factory.annotation.Autowired;
2929
import org.tron.common.error.TronDBException;
3030
import org.tron.common.es.ExecutorServiceManager;
31+
import org.tron.common.exit.ExitManager;
3132
import org.tron.common.parameter.CommonParameter;
3233
import org.tron.common.utils.FileUtil;
3334
import org.tron.common.utils.StorageUtils;
@@ -346,10 +347,10 @@ public void flush() {
346347
checkPointEnd - start,
347348
System.currentTimeMillis() - checkPointEnd
348349
);
349-
} catch (TronDBException e) {
350+
} catch (TronDBException | TronError e) {
350351
logger.error(" Find fatal error, program will be exited soon.", e);
351352
hitDown = true;
352-
throw new TronError(e, TronError.ErrCode.DB_FLUSH);
353+
throw ExitManager.requestExit(new TronError(e, TronError.ErrCode.DB_FLUSH));
353354
}
354355
}
355356
}

‎common/src/main/java/org/tron/common/exit/ExitManager.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,19 @@ public static Optional<TronError> findTronError(Throwable e) {
4444
return Optional.empty();
4545
}
4646

47+
public static TronError requestExit(TronError error) {
48+
logAndExit(error);
49+
return error;
50+
}
51+
4752
public static void logAndExit(TronError exit) {
53+
if (exit.isExitRequested()) {
54+
return;
55+
}
4856
final int code = exit.getErrCode().getCode();
4957
logger.error("Shutting down with code: {}, reason: {}", exit.getErrCode(), exit.getMessage());
5058
Thread exitThread = exitThreadFactory.newThread(() -> System.exit(code));
5159
exitThread.start();
60+
exit.markExitRequested();
5261
}
5362
}

‎common/src/main/java/org/tron/core/exception/TronError.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,20 @@
33
import lombok.Getter;
44

55
/**
6-
* If a {@link TronError} is thrown, the service will trigger {@link System#exit(int)} by
7-
* {@link Thread#setDefaultUncaughtExceptionHandler(Thread.UncaughtExceptionHandler)}.
8-
* NOTE: Do not attempt to catch {@link TronError}.
6+
* A {@link TronError} terminates the process only if it reaches the default uncaught exception
7+
* handler or an {@code ExecutorServiceManager} task wrapper. A site that runs after startup on a
8+
* thread whose errors reach neither (a netty pipeline, a Jetty request, a plain executor task)
9+
* must request the exit itself with {@code throw ExitManager.requestExit(...)}. Startup-only
10+
* sites throw directly so that the main thread unwinds before the process exits.
11+
* NOTE: Do not swallow {@link TronError}.
912
*/
1013
@Getter
1114
public class TronError extends Error {
1215

1316
private final ErrCode errCode;
1417

18+
private volatile boolean exitRequested;
19+
1520
public TronError(String message, ErrCode exitCode) {
1621
super(message);
1722
this.errCode = exitCode;
@@ -27,6 +32,10 @@ public TronError(Throwable cause, ErrCode exitCode) {
2732
this.errCode = exitCode;
2833
}
2934

35+
public void markExitRequested() {
36+
exitRequested = true;
37+
}
38+
3039
@Getter
3140
public enum ErrCode {
3241
WITNESS_KEYSTORE_LOAD(-1),

‎framework/src/main/java/org/tron/core/db/Manager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1402,7 +1402,7 @@ void blockTrigger(final BlockCapsule block, long oldSolid, long newSolid) {
14021402
} catch (Exception e) {
14031403
logger.error("Block trigger failed. head: {}, oldSolid: {}, newSolid: {}",
14041404
block.getNum(), oldSolid, newSolid, e);
1405-
throw new TronError(e, TronError.ErrCode.EVENT_SUBSCRIBE_ERROR);
1405+
throw ExitManager.requestExit(new TronError(e, TronError.ErrCode.EVENT_SUBSCRIBE_ERROR));
14061406
}
14071407
}
14081408

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package org.tron.common.exit;
2+
3+
import static org.junit.Assert.assertTrue;
4+
5+
import ch.qos.logback.classic.Logger;
6+
import ch.qos.logback.classic.spi.ILoggingEvent;
7+
import ch.qos.logback.core.read.ListAppender;
8+
import org.junit.Test;
9+
import org.slf4j.LoggerFactory;
10+
import org.tron.core.exception.TronError;
11+
12+
public class ExitManagerTest {
13+
14+
@Test
15+
public void logAndExitSkipsErrorWhoseExitWasRequested() {
16+
Logger exitLogger = (Logger) LoggerFactory.getLogger("Exit");
17+
ListAppender<ILoggingEvent> appender = new ListAppender<>();
18+
appender.start();
19+
exitLogger.addAppender(appender);
20+
try {
21+
TronError error = new TronError("already exiting", TronError.ErrCode.DB_FLUSH);
22+
error.markExitRequested();
23+
24+
ExitManager.logAndExit(error);
25+
26+
assertTrue(error.isExitRequested());
27+
assertTrue(appender.list.isEmpty());
28+
} finally {
29+
exitLogger.detachAppender(appender);
30+
}
31+
}
32+
}
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package org.tron.common.exit;
2+
3+
import static org.mockito.Mockito.mockStatic;
4+
5+
import org.mockito.MockedStatic;
6+
7+
public final class ExitManagerTestUtil {
8+
9+
private ExitManagerTestUtil() {
10+
}
11+
12+
public static MockedStatic<ExitManager> suppressExit() {
13+
return mockStatic(ExitManager.class, invocation ->
14+
"logAndExit".equals(invocation.getMethod().getName())
15+
? null : invocation.callRealMethod());
16+
}
17+
}

‎framework/src/test/java/org/tron/core/db/ManagerTest.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,13 @@
3939
import lombok.extern.slf4j.Slf4j;
4040
import org.junit.Assert;
4141
import org.junit.Test;
42+
import org.mockito.MockedStatic;
4243
import org.tron.api.GrpcAPI;
4344
import org.tron.common.BaseMethodTest;
4445
import org.tron.common.TestConstants;
4546
import org.tron.common.crypto.ECKey;
47+
import org.tron.common.exit.ExitManager;
48+
import org.tron.common.exit.ExitManagerTestUtil;
4649
import org.tron.common.logsfilter.EventPluginLoader;
4750
import org.tron.common.logsfilter.capsule.BlockFilterCapsule;
4851
import org.tron.common.logsfilter.capsule.BlockLogTriggerCapsule;
@@ -1441,9 +1444,12 @@ public void blockTrigger() {
14411444
Manager manager = spy(new Manager());
14421445
doThrow(new RuntimeException("postBlockTrigger mock")).when(manager)
14431446
.postBlockTrigger(any(), anyBoolean());
1444-
TronError thrown = Assert.assertThrows(TronError.class, () ->
1445-
manager.blockTrigger(new BlockCapsule(Block.newBuilder().build()), 1, 1));
1446-
Assert.assertEquals(TronError.ErrCode.EVENT_SUBSCRIBE_ERROR, thrown.getErrCode());
1447+
try (MockedStatic<ExitManager> exit = ExitManagerTestUtil.suppressExit()) {
1448+
TronError thrown = Assert.assertThrows(TronError.class, () ->
1449+
manager.blockTrigger(new BlockCapsule(Block.newBuilder().build()), 1, 1));
1450+
Assert.assertEquals(TronError.ErrCode.EVENT_SUBSCRIBE_ERROR, thrown.getErrCode());
1451+
exit.verify(() -> ExitManager.logAndExit(thrown));
1452+
}
14471453
}
14481454

14491455
@Test

‎framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.tron.core.db2;
22

3+
import static org.mockito.Mockito.doThrow;
34
import static org.mockito.Mockito.spy;
45
import static org.mockito.Mockito.when;
56

@@ -13,7 +14,10 @@
1314
import lombok.extern.slf4j.Slf4j;
1415
import org.junit.Assert;
1516
import org.junit.Test;
17+
import org.mockito.MockedStatic;
1618
import org.tron.common.BaseMethodTest;
19+
import org.tron.common.exit.ExitManager;
20+
import org.tron.common.exit.ExitManagerTestUtil;
1721
import org.tron.common.utils.Sha256Hash;
1822
import org.tron.core.capsule.BlockCapsule;
1923
import org.tron.core.db2.RevokingDbWithCacheNewValueTest.TestRevokingTronStore;
@@ -109,7 +113,25 @@ public void testFlushError() {
109113
manager.setUnChecked(false);
110114
when(manager.getCheckpointList()).thenReturn(Arrays.asList("check1", "check2"));
111115
when(manager.shouldBeRefreshed()).thenReturn(true);
112-
TronError thrown = Assert.assertThrows(TronError.class, manager::flush);
113-
Assert.assertEquals(TronError.ErrCode.DB_FLUSH, thrown.getErrCode());
116+
try (MockedStatic<ExitManager> exit = ExitManagerTestUtil.suppressExit()) {
117+
TronError thrown = Assert.assertThrows(TronError.class, manager::flush);
118+
Assert.assertEquals(TronError.ErrCode.DB_FLUSH, thrown.getErrCode());
119+
exit.verify(() -> ExitManager.logAndExit(thrown));
120+
}
121+
}
122+
123+
@Test
124+
public void testCheckpointOpenErrorRequestsExit() {
125+
SnapshotManager manager = spy(new SnapshotManager(""));
126+
manager.setUnChecked(false);
127+
when(manager.shouldBeRefreshed()).thenReturn(true);
128+
TronError openError = new TronError("open failed", TronError.ErrCode.LEVELDB_INIT);
129+
doThrow(openError).when(manager).createCheckpoint();
130+
try (MockedStatic<ExitManager> exit = ExitManagerTestUtil.suppressExit()) {
131+
TronError thrown = Assert.assertThrows(TronError.class, manager::flush);
132+
Assert.assertEquals(TronError.ErrCode.DB_FLUSH, thrown.getErrCode());
133+
Assert.assertSame(openError, thrown.getCause());
134+
exit.verify(() -> ExitManager.logAndExit(thrown));
135+
}
114136
}
115137
}

0 commit comments

Comments
 (0)