From 2300a43ed0f4b369e3dcd33a30dd3ef1ebf3fcf0 Mon Sep 17 00:00:00 2001 From: AlexTranAmz <167144297+AlexTranAmz@users.noreply.github.com> Date: Thu, 5 Nov 2026 03:52:56 +0000 Subject: [PATCH] fix: normalize Rust-emitted log record tags in openjd.sessions._v1 The Rust session runtime tags every record it emits on the "openjd.sessions" logger with session_id, openjd_log_content (a LogContent bitmask) and openjd_timestamp_usec. Those records reach Python through pyo3-log, which renders each structured value with its Rust Display implementation before passing it to Logger.makeRecord(..., extra=...), so openjd_log_content arrives as a decimal string such as "8" instead of the LogContent this logger documents. pyo3-log has no hook to preserve the types, and consumers that route records by type-checking openjd_log_content against LogContent dropped every Rust-emitted record. Install one logging.Filter on the "openjd.sessions" logger when openjd.sessions._v1 is imported. It converts openjd_log_content back to a LogContent and openjd_timestamp_usec back to an int. It sits on the logger rather than a handler because pyo3-log delivers records with Logger.handle(), which applies the logger's filters before any handler or handler-level filter sees the record. Values this LogContent cannot represent are left unchanged; the filter never drops a record. Also re-export LogContent from openjd.sessions._logging instead of declaring a second, identically-shaped Flag enum here: enum.Flag comparisons are class-scoped, so a record tagged with the duplicate class would fail the same type check consumers use. The end-to-end test runs a real Rust-backed session and is skipped until the installed openjd-model build forwards the structured fields. Signed-off-by: AlexTranAmz <167144297+AlexTranAmz@users.noreply.github.com> --- src/openjd/sessions/_v1/_logging.py | 79 ++++-- test/openjd/sessions_v1/test_logging.py | 268 ++++++++++++++++++ test/openjd/sessions_v1/test_rust_log_tags.py | 228 +++++++++++++++ 3 files changed, 552 insertions(+), 23 deletions(-) create mode 100644 test/openjd/sessions_v1/test_logging.py create mode 100644 test/openjd/sessions_v1/test_rust_log_tags.py diff --git a/src/openjd/sessions/_v1/_logging.py b/src/openjd/sessions/_v1/_logging.py index a9ebe6ca..38611cf5 100644 --- a/src/openjd/sessions/_v1/_logging.py +++ b/src/openjd/sessions/_v1/_logging.py @@ -2,33 +2,16 @@ from __future__ import annotations from typing import Any, MutableMapping, TypedDict -from enum import Flag, auto import logging -__all__ = ["LOG", "LoggerAdapter", "LogExtraInfo", "LogContent"] - +from .._logging import LogContent -class LogContent(Flag): - """ - A Flag Enum which describes the content of a log record. - """ +__all__ = ["LOG", "LoggerAdapter", "LogExtraInfo", "LogContent"] - BANNER = auto() # Logs which contain a banner, such as to visually seperate log sections - FILE_PATH = auto() # Logs which contain a filepath - FILE_CONTENTS = auto() # Logs which contain the contents of a file - COMMAND_OUTPUT = auto() # Logs which contain the output of a command run - EXCEPTION_INFO = ( - auto() - ) # Logs which contain an exception openjd encountered, potentially including sensitive information such as filepaths or host information. - PROCESS_CONTROL = ( - auto() - ) # Logs which contain information related to starting, killing, or signalling processes. - PARAMETER_INFO = ( - auto() - ) # Logs which contain details about parameters and their values pertaining to the running action - HOST_INFO = ( - auto() - ) # Logs which contain details about the system environment, e.g. dependency versions, OS name, CPU architecture. +# LogContent is the public openjd.sessions.LogContent class, not a second Flag enum +# with the same members. Consumers route records on the LOG logger by type-checking +# `record.openjd_log_content` against openjd.sessions.LogContent, and enum.Flag +# comparisons are class-scoped, so a duplicate class would fail that check. class LogExtraInfo(TypedDict): @@ -82,9 +65,59 @@ def process( If the LogRecord does not have the "openjd_log_content" no guarantees are made as to what content is in the LogRecord. LogRecords that contain LogContent.EXECPTION_INFO may also transitively include potentially sensitive information like filepaths or host info due to the nature of exception messages. + +LogRecords emitted by the Rust-backed session runtime additionally carry a "session_id" (str) and +an "openjd_timestamp_usec" (int). pyo3-log delivers all structured fields to Python as strings, so +importing openjd.sessions._v1 installs a filter on this logger that converts "openjd_log_content" +back to a LogContent and "openjd_timestamp_usec" back to an int before any handler sees the record. """ LOG.setLevel(logging.INFO) + +class _RustLogRecordNormalizingFilter(logging.Filter): + """Restores the Python types of the structured fields on Rust-emitted records. + + The Rust session runtime tags each record it emits with ``session_id`` (str), + ``openjd_log_content`` (a LogContent bitmask as an unsigned integer) and + ``openjd_timestamp_usec`` (unsigned integer). pyo3-log renders every value + with its Rust ``Display`` and passes the strings as ``extra`` to + ``Logger.makeRecord``, so ``openjd_log_content`` arrives as e.g. "8" and the + timestamp as a decimal string; pyo3-log has no hook to keep the types. + + Installed on the logger, not on a handler: pyo3-log delivers records with + ``Logger.handle()``, which runs the logger's filters before any handler (and + any handler-level filter) sees the record. Values this LogContent cannot + represent (e.g. a bit added by a newer runtime) are left as they arrived. + The filter never drops a record. + """ + + def filter(self, record: logging.LogRecord) -> bool: + # Rust formats these unsigned integers as plain decimal digits. Only + # convert that form: int() would also accept signs and whitespace, and + # a negative value would become a LogContent complement rather than an + # error. + content = getattr(record, "openjd_log_content", None) + if isinstance(content, str) and content.isdigit(): + try: + setattr(record, "openjd_log_content", LogContent(int(content))) + except ValueError: + pass + timestamp = getattr(record, "openjd_timestamp_usec", None) + if isinstance(timestamp, str) and timestamp.isdigit(): + setattr(record, "openjd_timestamp_usec", int(timestamp)) + return True + + +def _install_rust_log_record_normalizing_filter(logger: logging.Logger) -> None: + """Install one _RustLogRecordNormalizingFilter on ``logger``; idempotent.""" + if not any(isinstance(f, _RustLogRecordNormalizingFilter) for f in logger.filters): + logger.addFilter(_RustLogRecordNormalizingFilter()) + + +# openjd.sessions._v1 imports this module before the modules that load the Rust +# extension, so the filter is in place before the runtime emits its first record. +_install_rust_log_record_normalizing_filter(LOG) + _banner_log_extra = LogExtraInfo(openjd_log_content=LogContent.BANNER) diff --git a/test/openjd/sessions_v1/test_logging.py b/test/openjd/sessions_v1/test_logging.py new file mode 100644 index 00000000..ac4c8512 --- /dev/null +++ b/test/openjd/sessions_v1/test_logging.py @@ -0,0 +1,268 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the openjd.sessions._v1 logging module. + +The subject under test is the filter that ``openjd.sessions._v1._logging`` +installs on the "openjd.sessions" logger. pyo3-log delivers the Rust session +runtime's structured log fields to Python as strings, so ``openjd_log_content`` +arrives as a decimal bitmask string such as "8" and ``openjd_timestamp_usec`` as +a decimal string; the filter restores the documented Python types. +""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any, Iterator + +import pytest + +import openjd.sessions +import openjd.sessions._v1 +from openjd.sessions import LogContent +from openjd.sessions._v1._logging import ( + LOG, + _RustLogRecordNormalizingFilter, + _install_rust_log_record_normalizing_filter, +) + + +def _record(**extra: Any) -> logging.LogRecord: + return logging.makeLogRecord( + { + "name": LOG.name, + "msg": "message", + "levelno": logging.INFO, + "levelname": "INFO", + **extra, + } + ) + + +class _CollectingHandler(logging.Handler): + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +@pytest.fixture +def log_filter() -> _RustLogRecordNormalizingFilter: + return _RustLogRecordNormalizingFilter() + + +class TestNormalizingFilterLogContent: + @pytest.mark.parametrize( + "value, expected", + [ + *[(str(member.value), member) for member in LogContent], + ("10", LogContent.FILE_PATH | LogContent.COMMAND_OUTPUT), + ("0", LogContent(0)), + ], + ) + def test_decimal_string_becomes_log_content( + self, log_filter: _RustLogRecordNormalizingFilter, value: str, expected: LogContent + ) -> None: + # GIVEN + record = _record(openjd_log_content=value) + + # WHEN + result = log_filter.filter(record) + + # THEN + assert result is True + assert record.openjd_log_content is expected # type: ignore[attr-defined] + assert type(record.openjd_log_content) is openjd.sessions.LogContent # type: ignore[attr-defined] + + def test_already_typed_value_is_left_alone( + self, log_filter: _RustLogRecordNormalizingFilter + ) -> None: + # GIVEN + record = _record(openjd_log_content=LogContent.BANNER) + + # WHEN + log_filter.filter(record) + + # THEN + assert record.openjd_log_content is LogContent.BANNER # type: ignore[attr-defined] + + def test_missing_attribute_is_not_added( + self, log_filter: _RustLogRecordNormalizingFilter + ) -> None: + # GIVEN + record = _record() + + # WHEN + log_filter.filter(record) + + # THEN + assert not hasattr(record, "openjd_log_content") + assert not hasattr(record, "openjd_timestamp_usec") + + @pytest.mark.parametrize( + "value", + [ + "", + "abc", + "1.5", + "-1", + # A bit this LogContent does not define, e.g. from a newer runtime. + "4096", + None, + 8, + ], + ) + def test_unconvertible_value_is_preserved_unchanged( + self, log_filter: _RustLogRecordNormalizingFilter, value: Any + ) -> None: + # GIVEN + record = _record(openjd_log_content=value) + + # WHEN + result = log_filter.filter(record) + + # THEN + assert result is True + assert record.openjd_log_content == value # type: ignore[attr-defined] + assert type(record.openjd_log_content) is type(value) # type: ignore[attr-defined] + + +class TestNormalizingFilterTimestamp: + def test_digit_string_becomes_int(self, log_filter: _RustLogRecordNormalizingFilter) -> None: + # GIVEN + record = _record(openjd_timestamp_usec="1762288000123456") + + # WHEN + log_filter.filter(record) + + # THEN + assert record.openjd_timestamp_usec == 1762288000123456 # type: ignore[attr-defined] + assert type(record.openjd_timestamp_usec) is int # type: ignore[attr-defined] + + @pytest.mark.parametrize("value", [123, "", "soon", "1.5", None]) + def test_unconvertible_value_is_preserved_unchanged( + self, log_filter: _RustLogRecordNormalizingFilter, value: Any + ) -> None: + # GIVEN + record = _record(openjd_timestamp_usec=value) + + # WHEN + log_filter.filter(record) + + # THEN + assert record.openjd_timestamp_usec == value # type: ignore[attr-defined] + assert type(record.openjd_timestamp_usec) is type(value) # type: ignore[attr-defined] + + def test_session_id_is_not_touched(self, log_filter: _RustLogRecordNormalizingFilter) -> None: + # GIVEN + record = _record(session_id="123", openjd_log_content="8") + + # WHEN + log_filter.filter(record) + + # THEN + assert record.session_id == "123" # type: ignore[attr-defined] + + +class TestFilterInstallation: + def test_importing_v1_installs_exactly_one_normalizer_on_the_logger(self) -> None: + # GIVEN openjd.sessions._v1 is imported (module import above) + installed = [ + f + for f in logging.getLogger("openjd.sessions").filters + if isinstance(f, _RustLogRecordNormalizingFilter) + ] + + # THEN + assert len(installed) == 1 + + def test_repeated_install_does_not_stack(self) -> None: + # GIVEN + logger = logging.getLogger(f"openjd.sessions.test.{uuid.uuid4().hex}") + + # WHEN + _install_rust_log_record_normalizing_filter(logger) + _install_rust_log_record_normalizing_filter(logger) + + # THEN + assert len(logger.filters) == 1 + assert isinstance(logger.filters[0], _RustLogRecordNormalizingFilter) + + +class TestInstalledFilterOnTheLoggerHandlePath: + """pyo3-log hands records to ``Logger.handle()``, which applies the logger's + filters before any handler. Handlers, and filters on handlers, must see the + normalized record.""" + + @pytest.fixture + def handler(self) -> Iterator[_CollectingHandler]: + handler = _CollectingHandler() + LOG.addHandler(handler) + try: + yield handler + finally: + LOG.removeHandler(handler) + + def test_handler_sees_normalized_record(self, handler: _CollectingHandler) -> None: + # GIVEN + marker = uuid.uuid4().hex + record = _record( + msg=marker, openjd_log_content="32", openjd_timestamp_usec="7", session_id="s" + ) + + # WHEN + LOG.handle(record) + + # THEN + (seen,) = [r for r in handler.records if r.getMessage() == marker] + assert seen.openjd_log_content is LogContent.PROCESS_CONTROL # type: ignore[attr-defined] + assert seen.openjd_timestamp_usec == 7 # type: ignore[attr-defined] + assert seen.session_id == "s" # type: ignore[attr-defined] + + def test_handler_level_filter_sees_normalized_record(self, handler: _CollectingHandler) -> None: + # GIVEN + marker = uuid.uuid4().hex + seen_types: list[type] = [] + + class _TypeRecordingFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + if record.getMessage() == marker: + seen_types.append(type(getattr(record, "openjd_log_content", None))) + return True + + handler.addFilter(_TypeRecordingFilter()) + + # WHEN + LOG.handle(_record(msg=marker, openjd_log_content="128")) + + # THEN + assert seen_types == [LogContent] + + def test_untagged_record_still_reaches_the_handler(self, handler: _CollectingHandler) -> None: + # GIVEN + marker = uuid.uuid4().hex + + # WHEN + LOG.handle(_record(msg=marker)) + + # THEN + assert [r.getMessage() for r in handler.records if r.getMessage() == marker] == [marker] + + +class TestLogContentIsTheOnePublicClass: + def test_v1_log_content_is_the_public_class(self) -> None: + assert openjd.sessions._v1.LogContent is openjd.sessions.LogContent + + def test_members_and_bit_values_are_unchanged(self) -> None: + assert {m.name: m.value for m in LogContent} == { + "BANNER": 1, + "FILE_PATH": 2, + "FILE_CONTENTS": 4, + "COMMAND_OUTPUT": 8, + "EXCEPTION_INFO": 16, + "PROCESS_CONTROL": 32, + "PARAMETER_INFO": 64, + "HOST_INFO": 128, + } diff --git a/test/openjd/sessions_v1/test_rust_log_tags.py b/test/openjd/sessions_v1/test_rust_log_tags.py new file mode 100644 index 00000000..53893558 --- /dev/null +++ b/test/openjd/sessions_v1/test_rust_log_tags.py @@ -0,0 +1,228 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""End-to-end test of the Rust session runtime's structured log fields. + +Runs a real `openjd.sessions._v1.Session` task, captures the records that land +on the "openjd.sessions" logger, and checks that the fields the Rust session +runtime attaches to them arrive with the Python types that the logger +documents: a `LogContent` for `openjd_log_content`, the session's id as a `str` +for `session_id`, and an `int` for `openjd_timestamp_usec`. + +The fields only cross the Rust/Python boundary at all when the installed +openjd-model build enables pyo3-log's structured-logging support. Builds that +do not drop every structured field silently, so the tests here are gated on a +feature probe -- running one real session and looking for `session_id` on any +record. They skip on a build without the support and start passing on the +first build that has it, without a code change here. +""" + +from __future__ import annotations + +import logging +import sys +import time +import uuid +from pathlib import Path +from typing import Iterable, Iterator, NamedTuple + +import pytest + +from openjd.model._v1 import create_job, decode_job_template +from openjd.sessions import LogContent +from openjd.sessions._v1 import Session, SessionState + +LOGGER_NAME = "openjd.sessions" + +_TASK_OUTPUT = "openjd-rust-log-tag-probe" + +# `sys.executable -c ...` rather than a shell builtin such as `echo`, so the +# same command works on Linux, macOS and Windows. +_TEMPLATE = { + "specificationVersion": "jobtemplate-2023-09", + "name": "RustLogTags", + "steps": [ + { + "name": "Emit", + "script": { + "actions": { + "onRun": { + "command": sys.executable, + "args": ["-c", f"print({_TASK_OUTPUT!r})"], + } + } + }, + } + ], +} + +# Generous enough for a cold start of a real subprocess on a loaded CI worker, +# and well inside the suite's per-test timeout. +_ACTION_TIMEOUT_S = 15.0 +_QUIESCENT_STATES = (SessionState.READY, SessionState.ENDED, SessionState.READY_ENDING) + + +def records_carry_rust_tags(records: Iterable[logging.LogRecord]) -> bool: + """Whether `records` show that the Rust runtime's structured fields arrived. + + The runtime tags the records it emits with `session_id`, and that field + survives the trip into Python unchanged when the openjd-model build + forwards structured fields at all. Its presence on any record is therefore + the feature probe, and it is the one field that needs no normalization, so + the probe does not depend on the code under test. + """ + return any(hasattr(record, "session_id") for record in records) + + +class _CollectingHandler(logging.Handler): + """Keeps every record it is given, for inspection by a test.""" + + def __init__(self) -> None: + super().__init__(level=logging.DEBUG) + self.records: list[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + self.records.append(record) + + +class _SessionRun(NamedTuple): + session_id: str + records: list[logging.LogRecord] + + +def _run_one_task(session_root: Path) -> _SessionRun: + """Run one real task in a Rust-backed Session, returning what it logged.""" + job_template = decode_job_template(template=_TEMPLATE) + job = create_job(job_template=job_template, job_parameter_values={}) + step = job.steps[0] + + session_id = f"rust-log-tags-{uuid.uuid4().hex}" + handler = _CollectingHandler() + logger = logging.getLogger(LOGGER_NAME) + logger.addHandler(handler) + try: + with Session( + session_id=session_id, + job_parameter_values={}, + session_root_directory=session_root, + ) as session: + session.run_task( + step_script=step.script, + task_parameter_values={}, + resolved_symtab=step.resolved_symtab, + ) + deadline = time.monotonic() + _ACTION_TIMEOUT_S + while session.state not in _QUIESCENT_STATES and time.monotonic() < deadline: + time.sleep(0.05) + assert ( + session.state in _QUIESCENT_STATES + ), f"Session did not settle within {_ACTION_TIMEOUT_S}s; state={session.state}" + finally: + logger.removeHandler(handler) + + return _SessionRun(session_id=session_id, records=handler.records) + + +class TestFeatureProbe: + """The probe that gates the end-to-end tests below, tested on its own so the + gate cannot quietly stop working.""" + + def test_no_records_means_no_tags(self) -> None: + assert records_carry_rust_tags([]) is False + + def test_untagged_records_mean_no_tags(self) -> None: + records = [logging.makeLogRecord({"msg": "untagged"}) for _ in range(3)] + assert records_carry_rust_tags(records) is False + + def test_one_tagged_record_is_enough(self) -> None: + records = [ + logging.makeLogRecord({"msg": "untagged"}), + logging.makeLogRecord({"msg": "tagged", "session_id": "s"}), + ] + assert records_carry_rust_tags(records) is True + + +@pytest.fixture(scope="module") +def session_run(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SessionRun]: + """One real session run, shared by the tests that inspect its records.""" + yield _run_one_task(tmp_path_factory.mktemp("rust-log-tags")) + + +@pytest.fixture +def tagged_session_run(session_run: _SessionRun) -> _SessionRun: + """`session_run`, or a skip when this build does not forward the tags.""" + if not records_carry_rust_tags(session_run.records): + pytest.skip( + "The installed openjd-model build does not forward the Rust session " + "runtime's structured log fields to Python, so there is nothing to " + "normalize. This test runs against a build that does." + ) + return session_run + + +@pytest.mark.xdist_group("rust_log_tags") +class TestRustSessionLogTags: + """A real Rust-backed session's records reach consumers with the documented + Python types.""" + + def test_the_session_logged_something(self, session_run: _SessionRun) -> None: + """Ungated: a run that logged nothing would make every assertion below + vacuous, including the feature probe's.""" + assert session_run.records + assert any(_TASK_OUTPUT in record.getMessage() for record in session_run.records) + + def test_log_content_is_the_public_log_content_class( + self, tagged_session_run: _SessionRun + ) -> None: + # GIVEN + tagged = [ + record for record in tagged_session_run.records if hasattr(record, "openjd_log_content") + ] + + # THEN + assert tagged, "No record carried openjd_log_content" + for record in tagged: + assert isinstance(record.openjd_log_content, LogContent), ( # type: ignore[attr-defined] + f"openjd_log_content was {record.openjd_log_content!r} " # type: ignore[attr-defined] + f"on message {record.getMessage()!r}" + ) + + def test_session_id_identifies_this_session(self, tagged_session_run: _SessionRun) -> None: + # GIVEN + tagged = [record for record in tagged_session_run.records if hasattr(record, "session_id")] + + # THEN + assert tagged, "No record carried session_id" + for record in tagged: + assert record.session_id == tagged_session_run.session_id # type: ignore[attr-defined] + + def test_timestamp_is_an_int(self, tagged_session_run: _SessionRun) -> None: + # GIVEN + tagged = [ + record + for record in tagged_session_run.records + if hasattr(record, "openjd_timestamp_usec") + ] + + # THEN + assert tagged, "No record carried openjd_timestamp_usec" + for record in tagged: + assert type(record.openjd_timestamp_usec) is int # type: ignore[attr-defined] + assert record.openjd_timestamp_usec > 0 # type: ignore[attr-defined] + + def test_command_output_is_tagged_as_command_output( + self, tagged_session_run: _SessionRun + ) -> None: + """The task's stdout is what a consumer routes to the session log, so + check the bit the runtime sets on it specifically.""" + # GIVEN + output_records = [ + record for record in tagged_session_run.records if _TASK_OUTPUT in record.getMessage() + ] + + # THEN + assert output_records, "The task's output line was not captured" + assert any( + isinstance(getattr(record, "openjd_log_content", None), LogContent) + and LogContent.COMMAND_OUTPUT in record.openjd_log_content # type: ignore[attr-defined] + for record in output_records + )