Skip to content

Commit 85376bc

Browse files
authored
fix(net): reduce memory usage in transaction queues (#7011)
1 parent 5f9b426 commit 85376bc

2 files changed

Lines changed: 205 additions & 9 deletions

File tree

‎framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import org.tron.common.es.ExecutorServiceManager;
1919
import org.tron.common.utils.Sha256Hash;
2020
import org.tron.core.ChainBaseManager;
21+
import org.tron.core.Constant;
22+
import org.tron.core.capsule.TransactionCapsule;
2123
import org.tron.core.config.args.Args;
2224
import org.tron.core.exception.P2pException;
2325
import org.tron.core.exception.P2pException.TypeEnum;
@@ -88,31 +90,41 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep
8890
return;
8991
}
9092
TransactionsMessage transactionsMessage = (TransactionsMessage) msg;
91-
check(peer, transactionsMessage);
92-
for (Transaction trx : transactionsMessage.getTransactions().getTransactionsList()) {
93-
Item item = new Item(new TransactionMessage(trx).getMessageId(), InventoryType.TRX);
93+
Set<Sha256Hash> ids = check(peer, transactionsMessage);
94+
for (Sha256Hash id : ids) {
95+
Item item = new Item(id, InventoryType.TRX);
9496
peer.getAdvInvRequest().remove(item);
9597
}
9698
int smartContractQueueSize = 0;
9799
int trxHandlePoolQueueSize = 0;
98100
int dropSmartContractCount = 0;
101+
int oversizedCount = 0;
99102
for (Transaction trx : transactionsMessage.getTransactions().getTransactionsList()) {
100103
if (isClosed) {
101104
logger.info("TransactionsMsgHandler is closed during processing, stop submit");
102105
break;
103106
}
104-
int type = trx.getRawData().getContract(0).getType().getNumber();
107+
TransactionCapsule capsule = new TransactionCapsule(trx);
108+
capsule.sanitize();
109+
capsule.resetResult();
110+
if (capsule.getSerializedSize() > Constant.TRANSACTION_MAX_BYTE_SIZE) {
111+
oversizedCount++;
112+
continue;
113+
}
114+
// Both queues must retain the cleaned transaction, including its serialized bytes.
115+
Transaction cleanTrx = capsule.getInstance();
116+
int type = cleanTrx.getRawData().getContract(0).getType().getNumber();
105117
if (type == ContractType.TriggerSmartContract_VALUE
106118
|| type == ContractType.CreateSmartContract_VALUE) {
107-
if (!smartContractQueue.offer(new TrxEvent(peer, new TransactionMessage(trx)))) {
119+
if (!smartContractQueue.offer(new TrxEvent(peer, new TransactionMessage(cleanTrx)))) {
108120
smartContractQueueSize = smartContractQueue.size();
109121
trxHandlePoolQueueSize = queue.size();
110122
dropSmartContractCount++;
111123
}
112124
} else {
113125
try {
114126
ExecutorServiceManager.submit(
115-
trxHandlePool, () -> handleTransaction(peer, new TransactionMessage(trx)));
127+
trxHandlePool, () -> handleTransaction(peer, new TransactionMessage(cleanTrx)));
116128
} catch (RejectedExecutionException e) {
117129
logger.warn("Submit task to {} failed", trxEsName);
118130
break;
@@ -124,13 +136,17 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep
124136
logger.warn("Add smart contract failed, drop count: {}, queueSize {}:{}",
125137
dropSmartContractCount, smartContractQueueSize, trxHandlePoolQueueSize);
126138
}
139+
if (oversizedCount > 0) {
140+
logger.warn("Drop oversized transactions, count: {}", oversizedCount);
141+
}
127142
}
128143

129-
private void check(PeerConnection peer, TransactionsMessage msg) throws P2pException {
144+
private Set<Sha256Hash> check(PeerConnection peer, TransactionsMessage msg)
145+
throws P2pException {
130146
List<Transaction> list = msg.getTransactions().getTransactionsList();
131147
Set<Sha256Hash> seen = new HashSet<>(list.size() * 2);
132148
for (Transaction trx : list) {
133-
Sha256Hash id = new TransactionMessage(trx).getMessageId();
149+
Sha256Hash id = new TransactionCapsule(trx).getTransactionId();
134150
if (!seen.add(id)) {
135151
throw new P2pException(TypeEnum.BAD_MESSAGE,
136152
"TransactionsMessage contains duplicate transaction: " + id);
@@ -151,6 +167,7 @@ private void check(PeerConnection peer, TransactionsMessage msg) throws P2pExcep
151167
}
152168
}
153169
}
170+
return seen;
154171
}
155172

156173
private void handleSmartContract() {
@@ -216,4 +233,4 @@ public TrxEvent(PeerConnection peer, TransactionMessage msg) {
216233
this.time = System.currentTimeMillis();
217234
}
218235
}
219-
}
236+
}

‎framework/src/test/java/org/tron/core/net/messagehandler/TransactionsMsgHandlerTest.java‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22

33
import com.google.protobuf.Any;
44
import com.google.protobuf.ByteString;
5+
import com.google.protobuf.UnknownFieldSet;
56
import java.lang.reflect.Field;
67
import java.lang.reflect.Method;
78
import java.time.ZonedDateTime;
89
import java.util.ArrayList;
10+
import java.util.Collections;
911
import java.util.List;
1012
import java.util.Map;
1113
import java.util.concurrent.BlockingQueue;
@@ -25,7 +27,9 @@
2527
import org.tron.common.TestConstants;
2628
import org.tron.common.runtime.TvmTestUtils;
2729
import org.tron.common.utils.ByteArray;
30+
import org.tron.common.utils.ReflectUtils;
2831
import org.tron.core.ChainBaseManager;
32+
import org.tron.core.Constant;
2933
import org.tron.core.config.args.Args;
3034
import org.tron.core.exception.P2pException;
3135
import org.tron.core.exception.P2pException.TypeEnum;
@@ -199,6 +203,7 @@ public void testRejectedExecution() throws Exception {
199203
// 2 transfer transactions, submit throws on the first → catch + break, only called once
200204
handler.processMessage(peer, msg);
201205

206+
Assert.assertTrue(peer.getAdvInvRequest().isEmpty());
202207
Mockito.verify(mockPool, Mockito.times(1)).submit(Mockito.any(Runnable.class));
203208
} finally {
204209
closeHandlerAndOriginalPool(handler, originalPool);
@@ -389,11 +394,42 @@ public void testDuplicateTransactionRejected() throws Exception {
389394
} catch (P2pException e) {
390395
Assert.assertEquals(P2pException.TypeEnum.BAD_MESSAGE, e.getType());
391396
}
397+
Assert.assertEquals(1, advInvRequest.size());
398+
Assert.assertTrue(advInvRequest.containsKey(item));
392399
} finally {
393400
handler.close();
394401
}
395402
}
396403

404+
@Test
405+
public void testInvalidBatchPreservesRequests() throws Exception {
406+
TransactionsMsgHandler handler = new TransactionsMsgHandler();
407+
ExecutorService originalPool = null;
408+
try {
409+
ExecutorService pool = Mockito.mock(ExecutorService.class);
410+
originalPool = replaceTrxHandlePool(handler, pool);
411+
List<Protocol.Transaction> transactions = new ArrayList<>(
412+
buildTransferMessage(2).getTransactions().getTransactionsList());
413+
// The first transaction passes; the second fails signature-length validation.
414+
transactions.set(1, transactions.get(1).toBuilder()
415+
.addSignature(ByteString.copyFrom(new byte[64])).build());
416+
TransactionsMessage msg = new TransactionsMessage(transactions);
417+
PeerConnection peer = Mockito.mock(PeerConnection.class);
418+
stubAdvInvRequest(peer, msg);
419+
Map<Item, Long> requests = new ConcurrentHashMap<>(peer.getAdvInvRequest());
420+
Assert.assertEquals(2, requests.size());
421+
422+
P2pException exception = Assert.assertThrows(P2pException.class,
423+
() -> handler.processMessage(peer, msg));
424+
425+
Assert.assertEquals(TypeEnum.BAD_TRX, exception.getType());
426+
Assert.assertEquals(requests, peer.getAdvInvRequest());
427+
Mockito.verify(pool, Mockito.never()).submit(Mockito.any(Runnable.class));
428+
} finally {
429+
closeHandlerAndOriginalPool(handler, originalPool);
430+
}
431+
}
432+
397433
@Test
398434
public void testInvalidSigLength() throws Exception {
399435
TransactionsMsgHandler handler = new TransactionsMsgHandler();
@@ -492,6 +528,149 @@ public void testInvalidSigLength() throws Exception {
492528
}
493529
}
494530

531+
@Test
532+
public void testSanitizeBeforeQueueAdmission() throws Exception {
533+
for (boolean smart : new boolean[] {false, true}) {
534+
Protocol.Transaction base = admissionTransaction(smart);
535+
Protocol.Transaction clean = base.toBuilder().setRawData(base.getRawData().toBuilder()
536+
.setUnknownFields(padding(32))).build();
537+
Protocol.Transaction padded = clean.toBuilder().setUnknownFields(padding(600 * 1024))
538+
.build();
539+
assertAdmission(padded, clean);
540+
}
541+
}
542+
543+
@Test
544+
public void testOversizedTransactionDroppedBeforeQueueAdmission() throws Exception {
545+
for (boolean smart : new boolean[] {false, true}) {
546+
Protocol.Transaction base = admissionTransaction(smart);
547+
Protocol.Transaction oversized = base.toBuilder().setRawData(base.getRawData().toBuilder()
548+
.setData(ByteString.copyFrom(new byte[600 * 1024]))).build();
549+
assertAdmission(oversized, null);
550+
}
551+
}
552+
553+
@Test
554+
public void testQueueAdmissionSizeBoundary() throws Exception {
555+
int limit = (int) Constant.TRANSACTION_MAX_BYTE_SIZE;
556+
for (boolean smart : new boolean[] {false, true}) {
557+
Protocol.Transaction.raw.Builder raw = admissionTransaction(smart).getRawData().toBuilder()
558+
.setData(ByteString.copyFrom(new byte[limit]));
559+
int overhead = Protocol.Transaction.newBuilder().setRawData(raw).build()
560+
.getSerializedSize() - limit;
561+
Protocol.Transaction exact = Protocol.Transaction.newBuilder().setRawData(
562+
raw.setData(ByteString.copyFrom(new byte[limit - overhead]))).build();
563+
Assert.assertEquals(limit, exact.getSerializedSize());
564+
assertAdmission(exact, exact);
565+
Protocol.Transaction over = exact.toBuilder().setRawData(
566+
raw.setData(ByteString.copyFrom(new byte[limit - overhead + 1]))).build();
567+
Assert.assertEquals(limit + 1, over.getSerializedSize());
568+
assertAdmission(over, null);
569+
}
570+
}
571+
572+
@Test
573+
public void testResultsClearedBeforeQueueAdmission() throws Exception {
574+
for (boolean smart : new boolean[] {false, true}) {
575+
Protocol.Transaction expected = admissionTransaction(smart).toBuilder()
576+
.addSignature(ByteString.copyFrom(new byte[65])).build();
577+
// Even a single result must be discarded, including its known and unknown fields.
578+
Protocol.Transaction.Result result = Protocol.Transaction.Result.newBuilder()
579+
.setFee(123456).setWithdrawAmount(77)
580+
.setContractRet(Protocol.Transaction.Result.contractResult.REVERT)
581+
.setUnknownFields(padding(32)).build();
582+
assertAdmission(expected.toBuilder().addRet(result).build(), expected);
583+
assertAdmission(expected.toBuilder().addRet(result).addRet(result).build(), expected);
584+
}
585+
}
586+
587+
@Test
588+
public void testOversizedResultsClearedBeforeQueueAdmission() throws Exception {
589+
for (boolean smart : new boolean[] {false, true}) {
590+
Protocol.Transaction expected = admissionTransaction(smart).toBuilder()
591+
.addSignature(ByteString.copyFrom(new byte[65])).build();
592+
// Padding in the first result must be removed before the transaction size check.
593+
Protocol.Transaction padded = expected.toBuilder().addRet(
594+
Protocol.Transaction.Result.newBuilder().setUnknownFields(padding(600 * 1024))).build();
595+
Assert.assertTrue(padded.getSerializedSize() > Constant.TRANSACTION_MAX_BYTE_SIZE);
596+
assertAdmission(padded, expected);
597+
}
598+
}
599+
600+
private void assertAdmission(Protocol.Transaction input, Protocol.Transaction expected)
601+
throws Exception {
602+
TransactionsMsgHandler handler = new TransactionsMsgHandler();
603+
ExecutorService originalPool = null;
604+
try {
605+
ExecutorService pool = Mockito.mock(ExecutorService.class);
606+
List<Runnable> tasks = new ArrayList<>();
607+
Mockito.when(pool.submit(Mockito.any(Runnable.class))).thenAnswer(call -> {
608+
tasks.add(call.getArgument(0));
609+
return Mockito.mock(Future.class);
610+
});
611+
originalPool = replaceTrxHandlePool(handler, pool);
612+
TronNetDelegate delegate = Mockito.mock(TronNetDelegate.class);
613+
AdvService adv = Mockito.mock(AdvService.class);
614+
ReflectUtils.setFieldValue(handler, "tronNetDelegate", delegate);
615+
ReflectUtils.setFieldValue(handler, "advService", adv);
616+
ReflectUtils.setFieldValue(handler, "chainBaseManager", Mockito.mock(ChainBaseManager.class));
617+
PeerConnection peer = Mockito.mock(PeerConnection.class);
618+
TransactionsMessage message = new TransactionsMessage(Collections.singletonList(input));
619+
stubAdvInvRequest(peer, message);
620+
handler.processMessage(peer, message);
621+
BlockingQueue<?> smartQueue = (BlockingQueue<?>)
622+
ReflectUtils.getFieldObject(handler, "smartContractQueue");
623+
if (expected == null) {
624+
Assert.assertTrue(smartQueue.isEmpty());
625+
Mockito.verify(pool, Mockito.never()).submit(Mockito.any(Runnable.class));
626+
} else {
627+
TransactionMessage admitted;
628+
if (input.getRawData().getContract(0).getType()
629+
== Protocol.Transaction.Contract.ContractType.TriggerSmartContract) {
630+
Assert.assertEquals(1, smartQueue.size());
631+
admitted = (TransactionMessage) ReflectUtils.getFieldObject(smartQueue.peek(), "msg");
632+
} else {
633+
Assert.assertEquals(1, tasks.size());
634+
// Run only after admission completes, exercising the transaction retained by the task.
635+
tasks.get(0).run();
636+
org.mockito.ArgumentCaptor<TransactionMessage> captured =
637+
org.mockito.ArgumentCaptor.forClass(TransactionMessage.class);
638+
Mockito.verify(adv).broadcast(captured.capture());
639+
admitted = captured.getValue();
640+
}
641+
Protocol.Transaction retained = admitted.getTransactionCapsule().getInstance();
642+
Protocol.Transaction serialized = Protocol.Transaction.parseFrom(admitted.getData());
643+
Assert.assertEquals(expected, retained);
644+
Assert.assertEquals(0, retained.getRetCount());
645+
Assert.assertEquals(0, serialized.getRetCount());
646+
Assert.assertArrayEquals(input.getRawData().toByteArray(),
647+
retained.getRawData().toByteArray());
648+
Assert.assertEquals(input.getSignatureList(), retained.getSignatureList());
649+
Assert.assertArrayEquals(expected.toByteArray(), admitted.getData());
650+
Assert.assertEquals(new TransactionMessage(input).getMessageId(), admitted.getMessageId());
651+
}
652+
Assert.assertTrue(peer.getAdvInvRequest().isEmpty());
653+
Mockito.verify(peer, Mockito.never()).disconnect(Mockito.any());
654+
} finally {
655+
closeHandlerAndOriginalPool(handler, originalPool);
656+
}
657+
}
658+
659+
private Protocol.Transaction admissionTransaction(boolean smart) {
660+
if (smart) {
661+
return TvmTestUtils.generateTriggerSmartContractAndGetTransaction(
662+
ByteArray.fromHexString("121212a9cf"), ByteArray.fromHexString("121212a9cf"),
663+
ByteArray.fromHexString("123456"), 100, 100000000, 0, 0);
664+
}
665+
return buildTransferMessage(1).getTransactions().getTransactions(0);
666+
}
667+
668+
private UnknownFieldSet padding(int size) {
669+
return UnknownFieldSet.newBuilder().addField(99999,
670+
UnknownFieldSet.Field.newBuilder().addLengthDelimited(ByteString.copyFrom(new byte[size]))
671+
.build()).build();
672+
}
673+
495674
@Test
496675
public void testIsBusyWithCachedTransactions() throws Exception {
497676
TransactionsMsgHandler handler = new TransactionsMsgHandler();

0 commit comments

Comments
 (0)