Skip to content

Commit 18d2abf

Browse files
authored
fix: Eliminate intermittent failure in testQueueClosedEventTerminates… (#631)
…RemoteSubscribers Two-part fix for race condition in Kafka replication test: 1. EventQueue: Signal polling started BEFORE any early returns - Moved signalQueuePollerStarted() to beginning of dequeueEventItem() - Previously in finally block, which didn't execute if queue was closed - Race: If queue closed before first dequeue, EventQueueClosedException thrown BEFORE try block, so finally never ran and signal never sent - Now always signals even if queue immediately throws on closure 2. Test: Replace unreliable Thread.sleep with proper synchronization - Inject QueueManager to access the queue for the task - Replace Thread.sleep(2000) with queueManager.awaitQueuePollerStart() - Ensures EventConsumer is actually polling before sending QueueClosedEvent - Add debug logging to diagnose future failures Root cause: signalQueuePollerStarted() was only called in finally block, which was skipped when queue was already closed at first dequeue attempt. This caused awaitQueuePollerStart() to hang or timeout intermittently.
1 parent 767ca43 commit 18d2abf

2 files changed

Lines changed: 42 additions & 26 deletions

File tree

‎extras/queue-manager-replicated/tests-single-instance/src/test/java/io/a2a/extras/queuemanager/replicated/tests/KafkaReplicationIntegrationTest.java‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@
3030
import io.a2a.extras.queuemanager.replicated.core.ReplicatedEventQueueItem;
3131
import io.a2a.jsonrpc.common.json.JsonUtil;
3232
import io.a2a.server.PublicAgentCard;
33+
import io.a2a.server.events.EventQueue;
3334
import io.a2a.server.events.QueueClosedEvent;
35+
import io.a2a.server.events.QueueManager;
3436
import io.a2a.spec.A2AClientException;
3537
import io.a2a.spec.AgentCard;
3638
import io.a2a.spec.Message;
@@ -46,6 +48,8 @@
4648
import org.junit.jupiter.api.AfterEach;
4749
import org.junit.jupiter.api.BeforeEach;
4850
import org.junit.jupiter.api.Test;
51+
import org.slf4j.Logger;
52+
import org.slf4j.LoggerFactory;
4953

5054
/**
5155
* Integration test for Kafka replication functionality.
@@ -54,6 +58,8 @@
5458
@QuarkusTest
5559
public class KafkaReplicationIntegrationTest {
5660

61+
private static final Logger LOGGER = LoggerFactory.getLogger(KafkaReplicationIntegrationTest.class);
62+
5763
@Inject
5864
@PublicAgentCard
5965
AgentCard agentCard;
@@ -65,6 +71,9 @@ public class KafkaReplicationIntegrationTest {
6571
@Channel("replicated-events-out")
6672
Emitter<String> testEmitter;
6773

74+
@Inject
75+
QueueManager queueManager;
76+
6877
private Client streamingClient;
6978
private Client nonStreamingClient;
7079
private Client pollingClient;
@@ -367,8 +376,11 @@ public void testQueueClosedEventTerminatesRemoteSubscribers() throws Exception {
367376
// Resubscribe to the task - this creates a streaming subscription
368377
streamingClient.resubscribe(new TaskIdParams(taskId), List.of(consumer), errorHandler);
369378

370-
// Wait a moment to ensure the streaming subscription is fully established
371-
Thread.sleep(2000);
379+
// Wait for the EventConsumer to start polling (replaces unreliable Thread.sleep)
380+
// This ensures the consumer is ready to receive the QueueClosedEvent
381+
EventQueue queue = queueManager.get(taskId);
382+
assertNotNull(queue, "Queue should exist for task " + taskId);
383+
queueManager.awaitQueuePollerStart(queue);
372384

373385
// Now manually send a QueueClosedEvent to Kafka to simulate queue closure on another node
374386
QueueClosedEvent closedEvent = new QueueClosedEvent(taskId);
@@ -384,6 +396,10 @@ public void testQueueClosedEventTerminatesRemoteSubscribers() throws Exception {
384396
"Streaming subscription should complete when QueueClosedEvent is received");
385397

386398
// Verify the stream completed normally (not with an error)
399+
if (!streamCompleted.get()) {
400+
LOGGER.error("Stream did not complete normally! streamErrored={}, errorRef={}",
401+
streamErrored.get(), errorRef.get(), errorRef.get());
402+
}
387403
assertTrue(streamCompleted.get(), "Stream should complete normally when QueueClosedEvent is received");
388404
assertFalse(streamErrored.get(), "Stream should not error on QueueClosedEvent");
389405
assertNull(errorRef.get(), "Should not receive error when stream completes gracefully");

‎server-common/src/main/java/io/a2a/server/events/EventQueue.java‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -662,6 +662,10 @@ public void enqueueLocalOnly(EventQueueItem item) {
662662
@Override
663663
@Nullable
664664
public EventQueueItem dequeueEventItem(int waitMilliSeconds) throws EventQueueClosedException {
665+
// CRITICAL: Signal polling started BEFORE any early returns
666+
// This ensures awaitQueuePollerStart() unblocks even if queue is already closed
667+
signalQueuePollerStarted();
668+
665669
// For immediate close: exit immediately even if queue is not empty (race with MainEventBusProcessor)
666670
// For graceful close: only exit when queue is empty (wait for all events to be consumed)
667671
// BUT: if awaiting final event, keep polling even if closed and empty
@@ -672,32 +676,28 @@ public EventQueueItem dequeueEventItem(int waitMilliSeconds) throws EventQueueCl
672676
queue.size());
673677
throw new EventQueueClosedException();
674678
}
675-
try {
676-
if (waitMilliSeconds <= 0) {
677-
EventQueueItem item = queue.poll();
678-
if (item != null) {
679-
Event event = item.getEvent();
680-
LOGGER.debug("Dequeued event item (no wait) {} {}", this, event instanceof Throwable ? event.toString() : event);
681-
}
682-
return item;
679+
if (waitMilliSeconds <= 0) {
680+
EventQueueItem item = queue.poll();
681+
if (item != null) {
682+
Event event = item.getEvent();
683+
LOGGER.debug("Dequeued event item (no wait) {} {}", this, event instanceof Throwable ? event.toString() : event);
683684
}
684-
try {
685-
LOGGER.trace("Polling ChildQueue {} (wait={}ms)", System.identityHashCode(this), waitMilliSeconds);
686-
EventQueueItem item = queue.poll(waitMilliSeconds, TimeUnit.MILLISECONDS);
687-
if (item != null) {
688-
Event event = item.getEvent();
689-
LOGGER.debug("Dequeued event item (waiting) {} {}", this, event instanceof Throwable ? event.toString() : event);
690-
} else {
691-
LOGGER.trace("Dequeue timeout (null) from ChildQueue {}", System.identityHashCode(this));
692-
}
693-
return item;
694-
} catch (InterruptedException e) {
695-
LOGGER.debug("Interrupted dequeue (waiting) {}", this);
696-
Thread.currentThread().interrupt();
697-
return null;
685+
return item;
686+
}
687+
try {
688+
LOGGER.trace("Polling ChildQueue {} (wait={}ms)", System.identityHashCode(this), waitMilliSeconds);
689+
EventQueueItem item = queue.poll(waitMilliSeconds, TimeUnit.MILLISECONDS);
690+
if (item != null) {
691+
Event event = item.getEvent();
692+
LOGGER.debug("Dequeued event item (waiting) {} {}", this, event instanceof Throwable ? event.toString() : event);
693+
} else {
694+
LOGGER.trace("Dequeue timeout (null) from ChildQueue {}", System.identityHashCode(this));
698695
}
699-
} finally {
700-
signalQueuePollerStarted();
696+
return item;
697+
} catch (InterruptedException e) {
698+
LOGGER.debug("Interrupted dequeue (waiting) {}", this);
699+
Thread.currentThread().interrupt();
700+
return null;
701701
}
702702
}
703703

0 commit comments

Comments
 (0)