hand off log export to the hosted log sidecar if env var is set

livekit-agents/livekit/agents/job.py

@@ -46,6 +46,7 @@
     trace_types,
     utils as telemetry_utils,
 )
+from .telemetry.hosted_logs import logs_handled_by_sidecar
 from .telemetry.traces import (
     _BufferingHandler,
     _cloud_log_handler,
@@ -281,6 +282,8 @@ def _start_log_buffering(self) -> None:
             return
         if not _observability_url(self._info.url):
             return
+        if logs_handled_by_sidecar():
+            return
 
         self._early_log_handler = _BufferingHandler()
         logging.getLogger().addHandler(self._early_log_handler)

livekit-agents/livekit/agents/telemetry/hosted_logs.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import os
+
+from ..log import logger
+
+HOSTED_LOGS_ENV = "LIVEKIT_HOSTED_LOGS"
+HOSTED_LOGS_SIDECAR = "sidecar"
+HOSTED_LOGS_HANDOFF_FILE_ENV = "LIVEKIT_HOSTED_LOGS_HANDOFF_FILE"
+
+
+def _handoff_file() -> str | None:
+    if os.environ.get(HOSTED_LOGS_ENV) != HOSTED_LOGS_SIDECAR:
+        return None
+    return os.environ.get(HOSTED_LOGS_HANDOFF_FILE_ENV) or None
+
+
+def logs_handled_by_sidecar() -> bool:
+    path = _handoff_file()
+    return path is not None and os.path.exists(path)
+
+
+def hand_off_logs_to_sidecar() -> None:
+    path = _handoff_file()
+    if path is None:
+        return
+    try:
+        with open(path, "a"):
+            pass
+    except OSError:
+        logger.warning(
+            "could not create the hosted log hand-off file, the SDK keeps exporting logs",
+            extra={"path": path},
+            exc_info=True,
+        )

livekit-agents/livekit/agents/telemetry/traces.py

@@ -74,6 +74,7 @@
 )
 from ..utils import is_given
 from . import gen_ai, pii, trace_types, utils as telemetry_utils
+from .hosted_logs import logs_handled_by_sidecar
 
 if TYPE_CHECKING:
     from ..llm import ChatItem
@@ -973,7 +974,7 @@ def configure(
                 self._ensure_log_pipeline(url)
                 if self._log_metadata_processor is not None:
                     self._log_metadata_processor.set_metadata(session_metadata)
-                if self._log_handler is not None:
+                if self._log_handler is not None and not logs_handled_by_sidecar():
                     root = logging.getLogger()
                     if self._log_handler not in root.handlers:
                         root.addHandler(self._log_handler)

livekit-agents/livekit/agents/worker.py

@@ -52,6 +52,7 @@
 from .log import DEV_LEVEL, logger
 from .plugin import Plugin
 from .simulation import SimulationContext
+from .telemetry.hosted_logs import hand_off_logs_to_sidecar
 from .types import ATTRIBUTE_AGENT_NAME, NOT_GIVEN, NotGivenOr
 from .utils import http_context, http_server, is_given
 from .utils.hw import get_cpu_monitor
@@ -736,6 +737,7 @@ async def worker(_: Any) -> web.Response:
             # must run before job processes spawn so they inherit the SSL_CERT_FILE fallback
             http_context._set_default_cert_env()
 
+            hand_off_logs_to_sidecar()
             logger.info(
                 "starting worker",
                 extra={"version": __version__, "rtc-version": rtc.__version__},

tests/test_hosted_logs.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from livekit.agents.telemetry.hosted_logs import (
+    HOSTED_LOGS_ENV,
+    HOSTED_LOGS_HANDOFF_FILE_ENV,
+    HOSTED_LOGS_SIDECAR,
+    hand_off_logs_to_sidecar,
+    logs_handled_by_sidecar,
+)
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture
+def handoff_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
+    path = tmp_path / "sdk-handoff"
+    monkeypatch.setenv(HOSTED_LOGS_ENV, HOSTED_LOGS_SIDECAR)
+    monkeypatch.setenv(HOSTED_LOGS_HANDOFF_FILE_ENV, str(path))
+    return path
+
+
+def test_hand_off_creates_the_file(handoff_file: Path) -> None:
+    assert not logs_handled_by_sidecar()
+    hand_off_logs_to_sidecar()
+    assert handoff_file.exists()
+    assert logs_handled_by_sidecar()
+
+
+def test_hand_off_keeps_an_existing_file(handoff_file: Path) -> None:
+    handoff_file.write_text("x")
+    hand_off_logs_to_sidecar()
+    assert handoff_file.read_text() == "x"
+    assert logs_handled_by_sidecar()
+
+
+def test_no_hand_off_without_sidecar_mode(
+    handoff_file: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setenv(HOSTED_LOGS_ENV, "something-else")
+    hand_off_logs_to_sidecar()
+    assert not handoff_file.exists()
+    assert not logs_handled_by_sidecar()
+
+
+def test_no_hand_off_without_a_file_path(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv(HOSTED_LOGS_ENV, HOSTED_LOGS_SIDECAR)
+    monkeypatch.delenv(HOSTED_LOGS_HANDOFF_FILE_ENV, raising=False)
+    hand_off_logs_to_sidecar()
+    assert not logs_handled_by_sidecar()
+
+
+def test_sdk_keeps_exporting_when_the_file_cannot_be_created(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    monkeypatch.setenv(HOSTED_LOGS_ENV, HOSTED_LOGS_SIDECAR)
+    monkeypatch.setenv(HOSTED_LOGS_HANDOFF_FILE_ENV, str(tmp_path / "missing-dir" / "sdk-handoff"))
+    hand_off_logs_to_sidecar()
+    assert not logs_handled_by_sidecar()

tests/test_recording.py

@@ -1676,3 +1676,24 @@ async def test_recorder_io_not_created_when_audio_false() -> None:
 
     assert session._recorder_io is None
     await _cleanup(session)
+
+
+def test_setup_cloud_tracer_leaves_root_logger_to_the_log_sidecar(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    from livekit.agents.telemetry import traces as traces_mod
+    from livekit.agents.telemetry.hosted_logs import (
+        HOSTED_LOGS_ENV,
+        HOSTED_LOGS_HANDOFF_FILE_ENV,
+        HOSTED_LOGS_SIDECAR,
+    )
+
+    handoff_file = tmp_path / "sdk-handoff"
+    handoff_file.touch()
+    monkeypatch.setenv(HOSTED_LOGS_ENV, HOSTED_LOGS_SIDECAR)
+    monkeypatch.setenv(HOSTED_LOGS_HANDOFF_FILE_ENV, str(handoff_file))
+    with _stub_cloud_tracer_deps():
+        _setup_cloud_tracer_for_job()
+        cloud = traces_mod._cloud
+        assert cloud._log_batch_processor is not None
+        assert cloud.log_handler is None or cloud.log_handler not in logging.getLogger().handlers