[DO NOT MERGE] Probe hang stack dump against a live Kit process

conftest.py

@@ -19,15 +19,16 @@
 e.g. ``pytest -m unit source/isaaclab/test`` or ``pytest -m "not unit" source/isaaclab/test``.
 
 Also loads ``tools/ovrtx_log.py``, which replays the OVRTX renderer log per test, so every suite that
-builds a renderer reports what it logged the same way.
+builds a renderer reports what it logged the same way, and ``tools/hang_dump.py``, which lets the CI
+runner ask this process for a stack dump before it kills it for hanging.
 """
 
 from __future__ import annotations
 
 import json
 import os
 
-pytest_plugins = ["tools.ovrtx_log"]
+pytest_plugins = ["tools.ovrtx_log", "tools.hang_dump"]
 
 JOURNAL_ENV_VAR = "ISAACLAB_TEST_JOURNAL"
 """Environment variable naming the crash-journal file. Unset (the default) disables journaling."""

source/isaaclab/test/cli/test_test_orchestrator_result_handling.py

@@ -9,17 +9,31 @@
 
 import importlib.util
 import json
+import os
+import signal
 import sys
 import xml.etree.ElementTree as ElementTree
 from pathlib import Path
 from types import ModuleType
 
 import pytest
 
+TOOLS_DIR = Path(__file__).resolve().parents[4] / "tools"
+"""Repo ``tools/`` directory, holding the orchestrator and the stack-dump plugin it signals."""
+
+posix_only = pytest.mark.skipif(
+    not hasattr(signal, "SIGUSR1"),
+    reason="the orchestrator's process handling and the stack-dump signal are both POSIX-only",
+)
+"""Skip on platforms where ``capture_test_output_with_timeout`` cannot run.
+
+It needs ``select`` on pipes, ``os.killpg``, and ``start_new_session``, and the dump needs ``SIGUSR1``.
+"""
+
 
 def _load_orchestrator_module() -> ModuleType:
     """Load ``tools/conftest.py`` without registering it as a pytest plugin."""
-    module_path = Path(__file__).resolve().parents[4] / "tools" / "conftest.py"
+    module_path = TOOLS_DIR / "conftest.py"
     module_name = "isaaclab_test_orchestrator"
     tools_dir = str(module_path.parent)
     if tools_dir not in sys.path:
@@ -483,3 +497,92 @@ def test_result_summary_includes_fast_failure_after_thirty_slower_files():
     assert "Slowest 30 Test Files" not in summary
     assert "fast_failure.py" in summary
     assert all(test_path in summary for test_path in test_files)
+
+
+def _hanging_pytest_run(tmp_path: Path) -> tuple[list[str], dict[str, str]]:
+    """Return the command and environment for a hung *pytest* process.
+
+    The child has to be pytest, not a bare script. pytest captures at the file-descriptor level, so it has
+    already redirected fd 2 by the time a test runs; a dump written there is discarded when the process is
+    killed. A bare script has no such capture and would pass whether or not the dump reaches a file.
+    """
+    test_file = tmp_path / "test_wedges.py"
+    test_file.write_text(
+        "import threading\n\n\ndef test_wedges():\n    wedged_call()\n\n\ndef wedged_call():\n"
+        "    threading.Event().wait()\n",
+        encoding="utf-8",
+    )
+    env = os.environ.copy()
+    # `-p hang_dump` needs tools/ importable; the orchestrator gets this from the repo-root conftest.
+    env["PYTHONPATH"] = str(TOOLS_DIR) + os.pathsep + env.get("PYTHONPATH", "")
+    env["ISAACLAB_HANG_DUMP"] = str(tmp_path / "hangdump.log")
+    cmd = [sys.executable, "-m", "pytest", "-p", "hang_dump", "-p", "no:cacheprovider", str(test_file)]
+    return cmd, env
+
+
+@posix_only
+def test_hung_process_report_names_where_it_is_stuck(monkeypatch, tmp_path: Path) -> None:
+    """A hang must report the stack it is stuck in, not just that it stopped.
+
+    Without a dump the runner escalates straight to ``SIGKILL``, which cannot be caught, and the report
+    carries only system tables -- nothing that points at the hung code.
+    """
+    orchestrator = _load_orchestrator_module()
+    # The system tables are captured separately and are slow; this test is about the stack.
+    monkeypatch.setattr(orchestrator, "_capture_system_diagnostics", lambda: "")
+    # raising=False so a build without the dump still reaches the assertion below, and fails on the
+    # missing stack rather than on the missing constant.
+    monkeypatch.setattr(orchestrator, "HANG_DUMP_GRACE", 1, raising=False)
+
+    cmd, env = _hanging_pytest_run(tmp_path)
+    _returncode, _stdout, _stderr, kill_reason, _wall_time, pre_kill_diag = (
+        orchestrator.capture_test_output_with_timeout(cmd, timeout=15, env=env)
+    )
+
+    assert kill_reason == "timeout"
+    assert "HANG STACK DUMP" in pre_kill_diag
+    assert "wedged_call" in pre_kill_diag
+
+
+@posix_only
+def test_hung_process_is_dumped_more_than_once(monkeypatch, tmp_path: Path) -> None:
+    """Repeated dumps are what tell a wedged process apart from a slow one."""
+    orchestrator = _load_orchestrator_module()
+    monkeypatch.setattr(orchestrator, "_capture_system_diagnostics", lambda: "")
+    # raising=False so a build without the dump still reaches the assertion below, and fails on the
+    # missing stack rather than on the missing constant.
+    monkeypatch.setattr(orchestrator, "HANG_DUMP_GRACE", 1, raising=False)
+
+    cmd, env = _hanging_pytest_run(tmp_path)
+    *_, pre_kill_diag = orchestrator.capture_test_output_with_timeout(cmd, timeout=15, env=env)
+
+    assert pre_kill_diag.count("----- dump ") > 1
+
+
+@posix_only
+def test_hang_dump_precedes_system_diagnostics(monkeypatch, tmp_path: Path) -> None:
+    """The stack must sit ahead of the system tables, which ``_get_diagnostics`` truncates off the end."""
+    orchestrator = _load_orchestrator_module()
+    monkeypatch.setattr(orchestrator, "_capture_system_diagnostics", lambda: "=== SYSTEM DIAGNOSTICS BODY ===")
+    # raising=False so a build without the dump still reaches the assertion below, and fails on the
+    # missing stack rather than on the missing constant.
+    monkeypatch.setattr(orchestrator, "HANG_DUMP_GRACE", 1, raising=False)
+
+    cmd, env = _hanging_pytest_run(tmp_path)
+    *_, pre_kill_diag = orchestrator.capture_test_output_with_timeout(cmd, timeout=15, env=env)
+
+    assert "HANG STACK DUMP" in pre_kill_diag
+    assert pre_kill_diag.index("HANG STACK DUMP") < pre_kill_diag.index("SYSTEM DIAGNOSTICS BODY")
+
+
+def test_hang_dump_plugin_is_inert_without_signal_support(monkeypatch) -> None:
+    """The plugin loads on every platform, so it must no-op where the signal does not exist."""
+    if str(TOOLS_DIR) not in sys.path:
+        sys.path.insert(0, str(TOOLS_DIR))
+    import hang_dump
+
+    monkeypatch.setattr(hang_dump, "DUMP_SIGNAL", None)
+
+    assert hang_dump.is_supported() is False
+    assert hang_dump.register() is False
+    hang_dump.pytest_configure(config=None)  # must not raise

source/isaaclab_tasks/test/core/test_rendering_cartpole.py

@@ -31,7 +31,45 @@
 _attach_comparison_properties_fixture = make_attach_comparison_properties_fixture(_COMPARISON_SCORES)
 
 
+# ----------------------------------------------------------------------------------------------------
+# TEMPORARY CI PROBE -- DO NOT MERGE
+#
+# Wedges this file partway through the run so CI exercises the hang stack dump against a real Isaac Sim
+# process: Kit is fully up, has rendered actual frames, and all of its own threads are running when
+# SIGUSR1 arrives. The unit tests only prove the mechanism against a toy subprocess; this proves it
+# reports a live Kit process, whose main thread is parked in a native call.
+#
+# Expect: this file is killed for "timeout", and its job log and JUnit report carry a
+# "=== HANG STACK DUMP (all threads) ===" section naming _probe_wedge, twice.
+#
+# Revert by deleting this block and the _probe_wedge() call in test_rendering_cartpole.
+# ----------------------------------------------------------------------------------------------------
+import sys  # noqa: E402
+import threading  # noqa: E402
+
+_PROBE_CASES_BEFORE_WEDGE = 1
+"""Cases allowed to render normally before the process is wedged."""
+
+_probe_cases_done = 0
+
+
+def _probe_wedge():
+    """Block forever, once enough cases have actually rendered."""
+    global _probe_cases_done
+    _probe_cases_done += 1
+    if _probe_cases_done <= _PROBE_CASES_BEFORE_WEDGE:
+        return
+    print(
+        f"[CI PROBE] wedging after {_PROBE_CASES_BEFORE_WEDGE} rendered case(s);"
+        " expect a hang stack dump naming _probe_wedge",
+        file=sys.__stderr__,
+        flush=True,
+    )
+    threading.Event().wait()
+
+
 @pytest.mark.parametrize("physics_backend,renderer,data_type", PHYSICS_RENDERER_AOV_COMBINATIONS)
 def test_rendering_cartpole(physics_backend, renderer, data_type):
     """Test cartpole environment rendering correctness."""
     rendering_test_cartpole(physics_backend, renderer, data_type, _COMPARISON_SCORES, compare_golden=True)
+    _probe_wedge()  # TEMPORARY CI PROBE -- DO NOT MERGE