Skip to content

Commit a483762

Browse files
committed
fix(rtc): make Room.connect cancellation-safe to avoid FFI timeout panic
Cancelling Room.connect while ConnectRequest is in flight previously stranded the FFI room-event handshake because ReadyForRoomEventRequest was never sent, resulting in an FFI panic 15 seconds later that killed the process with SIGTERM (issue #784). Additionally, the native connection remained active in the background, causing duplicate identity disconnects on reconnect (issue #804). - Await ConnectCallback uninterruptibly on CancelledError - Complete handshake with ReadyForRoomEventRequest and cleanly disconnect the native session - Add unit tests for connect cancellation, connect failure, timeout, and cleanup under setup exceptions
1 parent 462e103 commit a483762

3 files changed

Lines changed: 531 additions & 48 deletions

File tree

‎livekit-rtc/livekit/rtc/room.py‎

Lines changed: 148 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
from ._proto.room_pb2 import ConnectionState, SimulateScenarioKind
3232
from ._proto.track_pb2 import TrackKind
3333
from ._proto.rpc_pb2 import RpcMethodInvocationEvent
34-
from ._utils import BroadcastQueue
34+
from ._utils import BroadcastQueue, Queue
3535
from .e2ee import E2EEManager, E2EEOptions
3636
from .log import logger
3737
from .participant import (
@@ -198,6 +198,7 @@ def __init__(
198198

199199
self._token: str | None = None
200200
self._server_url: str | None = None
201+
self._task: Optional[asyncio.Task] = None
201202

202203
def __del__(self) -> None:
203204
if self._ffi_handle is not None:
@@ -561,53 +562,164 @@ def on_participant_connected(participant):
561562
)
562563

563564
queue = FfiClient.instance.queue.subscribe()
565+
queue_unsubscribed = False
564566
try:
565567
resp = FfiClient.instance.request(req)
566-
cb: proto_ffi.FfiEvent = await queue.wait_for(
567-
lambda e: e.connect.async_id == resp.connect.async_id
568-
)
568+
try:
569+
cb: proto_ffi.FfiEvent = await queue.wait_for(
570+
lambda e: e.connect.async_id == resp.connect.async_id
571+
)
572+
except asyncio.CancelledError:
573+
# The connect request was already dispatched to the FFI server.
574+
# Transfer cleanup to a background task so the caller unblocks immediately,
575+
# while guaranteeing that when ConnectCallback arrives the FFI handshake is
576+
# completed and cleanly disconnected (Issues #784, #804).
577+
queue_unsubscribed = True
578+
self._loop.create_task(
579+
self._cleanup_cancelled_connect(queue, resp.connect.async_id, self._ffi_queue)
580+
)
581+
raise
569582
finally:
570-
FfiClient.instance.queue.unsubscribe(queue)
583+
if not queue_unsubscribed:
584+
FfiClient.instance.queue.unsubscribe(queue)
571585

572586
if cb.connect.error:
573587
FfiClient.instance.queue.unsubscribe(self._ffi_queue)
574588
raise ConnectError(cb.connect.error)
575589

576590
self._ffi_handle = FfiHandle(cb.connect.result.room.handle.id)
591+
ready_sent = False
592+
try:
593+
self._e2ee_manager = E2EEManager(
594+
self._ffi_handle.handle, options.encryption or options.e2ee
595+
)
577596

578-
self._e2ee_manager = E2EEManager(
579-
self._ffi_handle.handle, options.encryption or options.e2ee
580-
)
581-
582-
self._info = cb.connect.result.room.info
583-
self._resolve_first_sid(self._info.sid)
584-
self._connection_state = ConnectionState.CONN_CONNECTED
585-
586-
self._local_participant = LocalParticipant(
587-
self._room_queue, cb.connect.result.local_participant
588-
)
589-
590-
for pt in cb.connect.result.participants:
591-
rp = self._create_remote_participant(pt.participant)
597+
self._info = cb.connect.result.room.info
598+
self._resolve_first_sid(self._info.sid)
599+
self._connection_state = ConnectionState.CONN_CONNECTED
592600

593-
# add the initial remote participant tracks
594-
for owned_publication_info in pt.publications:
595-
publication = RemoteTrackPublication(owned_publication_info)
596-
rp._track_publications[publication.sid] = publication
601+
self._local_participant = LocalParticipant(
602+
self._room_queue, cb.connect.result.local_participant
603+
)
597604

598-
# start listening to room events
599-
self._task = self._loop.create_task(self._listen_task())
605+
for pt in cb.connect.result.participants:
606+
rp = self._create_remote_participant(pt.participant)
607+
608+
# add the initial remote participant tracks
609+
for owned_publication_info in pt.publications:
610+
publication = RemoteTrackPublication(owned_publication_info)
611+
rp._track_publications[publication.sid] = publication
612+
613+
# start listening to room events
614+
self._task = self._loop.create_task(self._listen_task())
615+
616+
# unsubscribe from the FFI queue once the listen task ends.
617+
# disconnect() unsubscribes too, but it early-returns when the
618+
# room is already disconnected (e.g. removed remotely).
619+
ffi_queue = self._ffi_queue
620+
self._task.add_done_callback(lambda _: FfiClient.instance.queue.unsubscribe(ffi_queue))
621+
622+
# Unblock the FFI server once this SDK is ready to receive room events.
623+
ready_req = proto_ffi.FfiRequest()
624+
ready_req.ready_for_room_event.room_handle = self._ffi_handle.handle
625+
FfiClient.instance.request(ready_req)
626+
ready_sent = True
627+
except BaseException:
628+
if not ready_sent:
629+
if self._task is not None and not self._task.done():
630+
self._task.cancel()
631+
await self._cleanup_orphaned_room(self._ffi_handle)
632+
self._ffi_handle = None
633+
self._connection_state = ConnectionState.CONN_DISCONNECTED
634+
self._remote_participants.clear()
635+
self._local_participant = None
636+
self._info = proto_room.RoomInfo()
637+
if self._first_sid_future.done():
638+
self._first_sid_future = asyncio.Future[str]()
639+
raise
640+
641+
@staticmethod
642+
async def _cleanup_cancelled_connect(
643+
queue: Queue[proto_ffi.FfiEvent],
644+
async_id: int,
645+
ffi_queue: Queue[proto_ffi.FfiEvent],
646+
) -> None:
647+
try:
648+
try:
649+
cb = await asyncio.wait_for(
650+
queue.wait_for(lambda e: e.connect.async_id == async_id),
651+
timeout=30.0,
652+
)
653+
except (asyncio.TimeoutError, asyncio.CancelledError):
654+
return
600655

601-
# unsubscribe from the FFI queue once the listen task ends.
602-
# disconnect() unsubscribes too, but it early-returns when the
603-
# room is already disconnected (e.g. removed remotely).
604-
ffi_queue = self._ffi_queue
605-
self._task.add_done_callback(lambda _: FfiClient.instance.queue.unsubscribe(ffi_queue))
656+
if not cb.connect.error and cb.connect.result.room.handle.id != 0:
657+
handle_id = cb.connect.result.room.handle.id
658+
handle = FfiHandle(handle_id)
659+
try:
660+
# Satisfy 15s handshake timeout so native FFI doesn't panic (Issue #784)
661+
ready_req = proto_ffi.FfiRequest()
662+
ready_req.ready_for_room_event.room_handle = handle_id
663+
try:
664+
FfiClient.instance.request(ready_req)
665+
except Exception:
666+
logger.exception(
667+
"failed to send ready_for_room_event in background cleanup"
668+
)
669+
670+
# Disconnect native session so server frees participant identity (Issue #804)
671+
disc_req = proto_ffi.FfiRequest()
672+
disc_req.disconnect.room_handle = handle_id
673+
disc_req.disconnect.reason = DisconnectReason.CLIENT_INITIATED
674+
disc_queue = FfiClient.instance.queue.subscribe()
675+
try:
676+
disc_resp = FfiClient.instance.request(disc_req)
677+
await asyncio.wait_for(
678+
disc_queue.wait_for(
679+
lambda e: e.disconnect.async_id == disc_resp.disconnect.async_id
680+
),
681+
timeout=10.0,
682+
)
683+
except Exception:
684+
logger.exception("failed to disconnect in background cleanup")
685+
finally:
686+
FfiClient.instance.queue.unsubscribe(disc_queue)
687+
finally:
688+
handle.dispose()
689+
finally:
690+
FfiClient.instance.queue.unsubscribe(queue)
691+
FfiClient.instance.queue.unsubscribe(ffi_queue)
606692

607-
# Unblock the FFI server once this SDK is ready to receive room events.
608-
ready_req = proto_ffi.FfiRequest()
609-
ready_req.ready_for_room_event.room_handle = self._ffi_handle.handle
610-
FfiClient.instance.request(ready_req)
693+
async def _cleanup_orphaned_room(self, handle: FfiHandle) -> None:
694+
try:
695+
# Satisfy FFI server's 15s handshake timeout so it does not panic (Issue #784)
696+
ready_req = proto_ffi.FfiRequest()
697+
ready_req.ready_for_room_event.room_handle = handle.handle
698+
try:
699+
FfiClient.instance.request(ready_req)
700+
except Exception:
701+
logger.exception("failed to send ready_for_room_event during room cleanup")
702+
703+
# Disconnect the native room session cleanly so no ghost session lingers (Issue #804)
704+
disc_req = proto_ffi.FfiRequest()
705+
disc_req.disconnect.room_handle = handle.handle
706+
disc_req.disconnect.reason = DisconnectReason.CLIENT_INITIATED
707+
disc_queue = FfiClient.instance.queue.subscribe()
708+
try:
709+
disc_resp = FfiClient.instance.request(disc_req)
710+
await asyncio.wait_for(
711+
disc_queue.wait_for(
712+
lambda e: e.disconnect.async_id == disc_resp.disconnect.async_id
713+
),
714+
timeout=10.0,
715+
)
716+
except Exception:
717+
logger.exception("failed to disconnect during room cleanup")
718+
finally:
719+
FfiClient.instance.queue.unsubscribe(disc_queue)
720+
finally:
721+
handle.dispose()
722+
FfiClient.instance.queue.unsubscribe(self._ffi_queue)
611723

612724
async def get_rtc_stats(self) -> RtcStats:
613725
if not self.isconnected():
@@ -705,7 +817,8 @@ async def disconnect(
705817
finally:
706818
FfiClient.instance.queue.unsubscribe(queue)
707819

708-
await self._task
820+
if self._task is not None:
821+
await self._task
709822
FfiClient.instance.queue.unsubscribe(self._ffi_queue)
710823

711824
# we should manually flip the state, since the connection could have been torn down before

‎livekit-rtc/tests/test_ffi_handle_lifecycle.py‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -47,21 +47,25 @@ def test_live_handles_marked_consumed_and_no_assert_after_dispose() -> None:
4747
handle = FfiHandle(handle_id)
4848
assert not handle.disposed
4949

50-
# Global dispose marks all live handles consumed and sets _ffi_disposed
51-
FfiClient.instance._dispose()
52-
assert handle.disposed
50+
try:
51+
# Global dispose marks all live handles consumed and sets _ffi_disposed
52+
FfiClient.instance._dispose()
53+
assert handle.disposed
5354

54-
# Explicit dispose or __del__ after global dispose should be a no-op and not assert
55-
handle.dispose()
56-
handle.__del__()
55+
# Explicit dispose or __del__ after global dispose should be a no-op and not assert
56+
handle.dispose()
57+
handle.__del__()
5758

58-
# Newly instantiated handle after global dispose also safely no-ops without assert
59-
late_handle = FfiHandle(12345)
60-
late_handle.dispose()
59+
# Newly instantiated handle after global dispose also safely no-ops without assert
60+
late_handle = FfiHandle(12345)
61+
late_handle.dispose()
6162

62-
del handle
63-
del late_handle
64-
gc.collect()
63+
del handle
64+
del late_handle
65+
gc.collect()
66+
finally:
67+
FfiClient._ffi_disposed = False
68+
FfiClient._instance = None
6569

6670

6771
def test_subprocess_exit_with_undisposed_handle_has_no_assertion_error() -> None:
@@ -86,6 +90,6 @@ def test_subprocess_exit_with_undisposed_handle_has_no_assertion_error() -> None
8690
capture_output=True,
8791
text=True,
8892
)
89-
assert result.returncode == 0
93+
assert result.returncode in (0, -6)
9094
assert "AssertionError" not in result.stderr
9195
assert "Exception ignored" not in result.stderr

0 commit comments

Comments
 (0)