From 99d53720107f6e0caa3f6695b5df402efbddb78d Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Wed, 12 Aug 2026 14:15:18 -0700 Subject: [PATCH 1/6] Raise the paused-frame channel threshold for Kit RTX visualizer tests RTX ResponsiveDenoising keeps refining a paused frame, so two captures taken while simulation is frozen differ by a small number of high-amplitude pixels. Amplitude separates that residue from real motion and the pixel count does not: at the shared threshold of 50 the paused noise reaches 1.55x the weakest real motion signal, so any count-based gate loose enough to stay quiet is also blind to a genuine regression. _assert_frames_remain_stable now takes a channel_diff_threshold, raised to 80 at the Kit viewport pause and 150 at the Kit tiled camera pause. The tiled override is gated on KitVisualizer because all four tiled cases share _attempt_pause and the Newton-visualizer cells sit at zero noise. Newton ViewerGL rasterises without DLSS and keeps the strict default. The pixel gate stays at 100. Both paused tiled captures render fresh again, reverting #6658. Comparing the sensor's cached frame against itself cannot detect a renderer that keeps changing the image after physics stops, which left that assertion unable to fail. Measured on L40 / driver 595.58.03 / Isaac Sim 6.1.0-alpha.47. Work-around for NVBUG 6570125; remove once OVRTX 0.5 ships the fix. --- .../fanes-nvbug-6570125-adjust-threshold.rst | 17 ++++++++ .../test/visualizer_integration_utils.py | 41 ++++++++++++++----- 2 files changed, 47 insertions(+), 11 deletions(-) create mode 100644 source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst diff --git a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst new file mode 100644 index 000000000000..ad5703710f48 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst @@ -0,0 +1,17 @@ +Changed +^^^^^^^ + +* Changed ``_assert_frames_remain_stable`` in the visualizer integration tests to accept a + ``channel_diff_threshold``, and raised it for the two RTX pause comparisons only: 80 for the Kit + viewport and 150 for the Kit tiled camera. RTX ResponsiveDenoising keeps refining a paused frame, + so the residue is a few high-amplitude pixels that a per-pixel count cannot separate from real + motion. The Newton ViewerGL pause checks rasterise without DLSS and keep the strict default of 50. + This is a work-around for NVBUG 6570125 and should be reverted once OVRTX 0.5 ships the fix. + +Fixed +^^^^^ + +* Fixed the tiled camera pause assertion comparing the sensor's cached frame against itself, which + made the check vacuous — it could not fail regardless of what the renderer did while paused. Both + paused captures force a fresh render again, and the simulation app is pumped during the pause + window. diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py index 17c4351ad927..05379147eabd 100644 --- a/source/isaaclab_visualizers/test/visualizer_integration_utils.py +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -175,6 +175,20 @@ _TILED_CAMERA_MOTION_MIN_DIFFERING_PIXELS = 25 """Minimum differing pixels for tiled camera motion checks.""" +# NVBUG 6570125 — RTX ResponsiveDenoising (turned on by Kit 9e5dd32c, driver-gated at R580+) keeps +# refining the denoised image after physics stops, so two captures taken while paused differ by a +# small number of high-amplitude pixels. Amplitude separates that residue from real motion and the +# pixel count does not: at the shared threshold of 50 the paused noise reaches 1.55x the weakest real +# motion signal, so any count-based gate loose enough to be quiet is also blind. Measured on L40 / +# driver 595.58.03 / Isaac Sim 6.1.0-alpha.47. The Newton ViewerGL pause checks rasterise without +# DLSS, see none of this, and must keep the strict default. Remove both overrides once OVRTX 0.5 +# ships the fix and paused frames are stable again. +_KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD = 80 +"""Per-channel threshold for paused Kit viewport comparisons (0–255 space).""" + +_KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD = 150 +"""Per-channel threshold for paused Kit tiled camera comparisons (0–255 space).""" + _FRAME_MIN_CHANNEL_RANGE = 10 """Minimum per-frame channel range to reject all-one-color images.""" @@ -569,12 +583,14 @@ def _assert_frames_remain_stable( phase: str, debug_phase: str, max_differing_pixels: int = 100, + channel_diff_threshold: float = _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD, ) -> None: """Assert two viewport frames are effectively unchanged while simulation is paused.""" - n_diff = _count_significantly_differing_pixels(frame_a, frame_b) + n_diff = _count_significantly_differing_pixels(frame_a, frame_b, channel_diff_threshold=channel_diff_threshold) assert n_diff <= max_differing_pixels, ( f"{case_label} failed to pause during {phase}: {n_diff} pixels differed, expected at most " - f"{max_differing_pixels}. Frame shape={_frame_shape_for_message(frame_a)}. " + f"{max_differing_pixels} with per-channel threshold {channel_diff_threshold} in 0-255 space. " + f"Frame shape={_frame_shape_for_message(frame_a)}. " f"Debug frames: {_current_visualizer_debug_dir()}/*{debug_phase}*.png." ) @@ -1259,6 +1275,7 @@ def _attempt_kit_pause(): case_label=case_label, phase="pausing", debug_phase="pausing", + channel_diff_threshold=_KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD, ) try: @@ -1400,16 +1417,13 @@ def _run_visualizer_tiled_camera_motion_test(env, visualizer, *, physics_kind: s def _attempt_pause(): _set_kit_simulation_paused(env, True) - # Read the sensor's last completed frame while paused. Forcing a new RTX - # render here introduces TAA edge jitter even though physics is frozen. - paused_start_frame = _capture_visualizer_tiled_camera_rgb( - visualizer, label="2a_pausing_frame_20", force_recompute=False - ) + # Re-render both paused captures: comparing the sensor's cached frame with itself cannot + # detect a renderer that keeps changing the image after physics stops. The denoiser residue + # that motivated caching is handled by the per-channel threshold below (NVBUG 6570125). + paused_start_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="2a_pausing_frame_20") for _ in range(PAUSE_VIZ_N_STEP): - env.sim.render(skip_app_pumping=isinstance(visualizer, KitVisualizer)) - paused_end_frame = _capture_visualizer_tiled_camera_rgb( - visualizer, label="2b_pausing_frame_25", force_recompute=False - ) + env.sim.render() + paused_end_frame = _capture_visualizer_tiled_camera_rgb(visualizer, label="2b_pausing_frame_25") _save_visualizer_debug_phase_images( paused_start_frame, paused_end_frame, @@ -1425,6 +1439,11 @@ def _attempt_pause(): case_label=case_label, phase="pausing", debug_phase="pausing_tiled", + channel_diff_threshold=( + _KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD + if isinstance(visualizer, KitVisualizer) + else _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD + ), ) try: From c793715fb74b6422cd9ea1b69bca44cae2e6c847 Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Wed, 12 Aug 2026 15:49:38 -0700 Subject: [PATCH 2/6] Re-enable the Kit pause checks with amplitude-based thresholds These checks have been dead weight for a while. RTX responsive denoising keeps refining the image after physics stops, so two frames captured during a pause differ by ~900 pixels and the test went red constantly. #6658 quietened it by comparing the camera's cached frame against itself, which can't fail whatever the renderer does. The denoiser residue is a few very bright pixels; real motion is spread across many. So count on amplitude instead: only pixels differing by 160 or more (80 in the viewport). The shimmer drops out, real movement doesn't. Measured over 24 runs on an L40, and confirmed the check still fails when fed actual motion. Worth being clear about the scope: no physics can step inside this pause window, so this is a renderer-stability check, not a pause-correctness one. It catches a gross instability, not a subtle one. Work-around/Temp fix for NVBUG 6570125. Need to remove once the real fix is in. --- .../changelog.d/fanes-nvbug-6570125-adjust-threshold.rst | 2 +- .../test/visualizer_integration_utils.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst index ad5703710f48..a875c66a4aa3 100644 --- a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst +++ b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst @@ -3,7 +3,7 @@ Changed * Changed ``_assert_frames_remain_stable`` in the visualizer integration tests to accept a ``channel_diff_threshold``, and raised it for the two RTX pause comparisons only: 80 for the Kit - viewport and 150 for the Kit tiled camera. RTX ResponsiveDenoising keeps refining a paused frame, + viewport and 160 for the Kit tiled camera. RTX ResponsiveDenoising keeps refining a paused frame, so the residue is a few high-amplitude pixels that a per-pixel count cannot separate from real motion. The Newton ViewerGL pause checks rasterise without DLSS and keep the strict default of 50. This is a work-around for NVBUG 6570125 and should be reverted once OVRTX 0.5 ships the fix. diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py index 05379147eabd..de9d1cf2ad4f 100644 --- a/source/isaaclab_visualizers/test/visualizer_integration_utils.py +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -180,13 +180,14 @@ # small number of high-amplitude pixels. Amplitude separates that residue from real motion and the # pixel count does not: at the shared threshold of 50 the paused noise reaches 1.55x the weakest real # motion signal, so any count-based gate loose enough to be quiet is also blind. Measured on L40 / -# driver 595.58.03 / Isaac Sim 6.1.0-alpha.47. The Newton ViewerGL pause checks rasterise without -# DLSS, see none of this, and must keep the strict default. Remove both overrides once OVRTX 0.5 -# ships the fix and paused frames are stable again. +# driver 595.58.03 / Isaac Sim 6.1.0-alpha.47. Each override straddles the 100-pixel gate evenly: +# at 160 the tiled cell measures 49 paused pixels against 213 for real motion over 24 runs. The +# Newton ViewerGL pause checks rasterise without DLSS, see none of this, and must keep the strict +# default. Remove both overrides once OVRTX 0.5 ships the fix and paused frames are stable again. _KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD = 80 """Per-channel threshold for paused Kit viewport comparisons (0–255 space).""" -_KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD = 150 +_KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD = 160 """Per-channel threshold for paused Kit tiled camera comparisons (0–255 space).""" _FRAME_MIN_CHANNEL_RANGE = 10 From 4d7132d79c0312fdb405985392350c1bfba22ce8 Mon Sep 17 00:00:00 2001 From: fanes <74020209+fatimaanes@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:56:37 -0700 Subject: [PATCH 3/6] update comment Shortened the comment regarding NVBUG 6570125 for clarity. Signed-off-by: fanes <74020209+fatimaanes@users.noreply.github.com> --- .../test/visualizer_integration_utils.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py index de9d1cf2ad4f..8b20c68debf3 100644 --- a/source/isaaclab_visualizers/test/visualizer_integration_utils.py +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -175,15 +175,7 @@ _TILED_CAMERA_MOTION_MIN_DIFFERING_PIXELS = 25 """Minimum differing pixels for tiled camera motion checks.""" -# NVBUG 6570125 — RTX ResponsiveDenoising (turned on by Kit 9e5dd32c, driver-gated at R580+) keeps -# refining the denoised image after physics stops, so two captures taken while paused differ by a -# small number of high-amplitude pixels. Amplitude separates that residue from real motion and the -# pixel count does not: at the shared threshold of 50 the paused noise reaches 1.55x the weakest real -# motion signal, so any count-based gate loose enough to be quiet is also blind. Measured on L40 / -# driver 595.58.03 / Isaac Sim 6.1.0-alpha.47. Each override straddles the 100-pixel gate evenly: -# at 160 the tiled cell measures 49 paused pixels against 213 for real motion over 24 runs. The -# Newton ViewerGL pause checks rasterise without DLSS, see none of this, and must keep the strict -# default. Remove both overrides once OVRTX 0.5 ships the fix and paused frames are stable again. +# NVBUG 6570125 — Remove both overrides once it ships the fix and paused frames are stable again. _KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD = 80 """Per-channel threshold for paused Kit viewport comparisons (0–255 space).""" From 8b2a11fb4dcbf8dbd20d9ffba44a3e7b84b59b75 Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Wed, 12 Aug 2026 16:09:51 -0700 Subject: [PATCH 4/6] Update comment --- .../changelog.d/fanes-nvbug-6570125-adjust-threshold.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst index a875c66a4aa3..bcb9b28ee3ef 100644 --- a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst +++ b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst @@ -6,7 +6,7 @@ Changed viewport and 160 for the Kit tiled camera. RTX ResponsiveDenoising keeps refining a paused frame, so the residue is a few high-amplitude pixels that a per-pixel count cannot separate from real motion. The Newton ViewerGL pause checks rasterise without DLSS and keep the strict default of 50. - This is a work-around for NVBUG 6570125 and should be reverted once OVRTX 0.5 ships the fix. + This is a work-around for NVBUG 6570125 and should be reverted once the renderer fix ships. Fixed ^^^^^ From 8e5cab8d63ff91d4100efa0c314496f29aeec096 Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Thu, 13 Aug 2026 11:10:49 -0700 Subject: [PATCH 5/6] Log pause pixel count; tighten tiled threshold scope --- .../test/visualizer_integration_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py index 8b20c68debf3..59ba1650a17b 100644 --- a/source/isaaclab_visualizers/test/visualizer_integration_utils.py +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -580,6 +580,13 @@ def _assert_frames_remain_stable( ) -> None: """Assert two viewport frames are effectively unchanged while simulation is paused.""" n_diff = _count_significantly_differing_pixels(frame_a, frame_b, channel_diff_threshold=channel_diff_threshold) + logging.getLogger(__name__).info( + "%s pause stability: %d px differed at threshold %g (gate %d)", + case_label, + n_diff, + channel_diff_threshold, + max_differing_pixels, + ) assert n_diff <= max_differing_pixels, ( f"{case_label} failed to pause during {phase}: {n_diff} pixels differed, expected at most " f"{max_differing_pixels} with per-channel threshold {channel_diff_threshold} in 0-255 space. " @@ -1434,6 +1441,8 @@ def _attempt_pause(): debug_phase="pausing_tiled", channel_diff_threshold=( _KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD + if isinstance(visualizer, KitVisualizer) and physics_kind == "newton" + else _KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD if isinstance(visualizer, KitVisualizer) else _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD ), From 538349c64d2d2a02f32d24791189219a02c3d815 Mon Sep 17 00:00:00 2001 From: Fatima Anes Date: Thu, 13 Aug 2026 13:28:25 -0700 Subject: [PATCH 6/6] Drop dead pause log and split the tiled PhysX threshold The pause pixel-count log never reaches CI: every pause call site runs inside caplog.at_level(logging.WARNING), which hard-sets the root logger, so the module logger's effective level is WARNING and info() short-circuits. Raising it to WARNING would instead self-trip _assert_no_visualizer_log_issues once ASSERT_VISUALIZER_WARNINGS is enabled. The assert message already carries the count, threshold and gate on the run where it matters. Give the tiled PhysX cell its own constant rather than borrowing the viewport one. It measures 103 differing pixels at the default threshold against a gate of 100, so lowering the viewport value later would silently break it. --- .../fanes-nvbug-6570125-adjust-threshold.rst | 9 +++---- .../test/visualizer_integration_utils.py | 24 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst index bcb9b28ee3ef..3e2c4b307dd4 100644 --- a/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst +++ b/source/isaaclab_visualizers/changelog.d/fanes-nvbug-6570125-adjust-threshold.rst @@ -2,10 +2,11 @@ Changed ^^^^^^^ * Changed ``_assert_frames_remain_stable`` in the visualizer integration tests to accept a - ``channel_diff_threshold``, and raised it for the two RTX pause comparisons only: 80 for the Kit - viewport and 160 for the Kit tiled camera. RTX ResponsiveDenoising keeps refining a paused frame, - so the residue is a few high-amplitude pixels that a per-pixel count cannot separate from real - motion. The Newton ViewerGL pause checks rasterise without DLSS and keep the strict default of 50. + ``channel_diff_threshold``, and raised it for the Kit RTX pause comparisons only: 80 for the Kit + viewport, 160 for the Kit tiled camera on Newton, and 80 for the Kit tiled camera on PhysX. RTX + ResponsiveDenoising keeps refining a paused frame, so the residue is a few high-amplitude pixels + that a per-pixel count cannot separate from real motion. The Newton ViewerGL pause checks + rasterise without DLSS and keep the strict default of 50. This is a work-around for NVBUG 6570125 and should be reverted once the renderer fix ships. Fixed diff --git a/source/isaaclab_visualizers/test/visualizer_integration_utils.py b/source/isaaclab_visualizers/test/visualizer_integration_utils.py index 59ba1650a17b..51fb39088096 100644 --- a/source/isaaclab_visualizers/test/visualizer_integration_utils.py +++ b/source/isaaclab_visualizers/test/visualizer_integration_utils.py @@ -175,12 +175,19 @@ _TILED_CAMERA_MOTION_MIN_DIFFERING_PIXELS = 25 """Minimum differing pixels for tiled camera motion checks.""" -# NVBUG 6570125 — Remove both overrides once it ships the fix and paused frames are stable again. +# NVBUG 6570125 — Remove these overrides once it ships the fix and paused frames are stable again. _KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD = 80 """Per-channel threshold for paused Kit viewport comparisons (0–255 space).""" -_KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD = 160 -"""Per-channel threshold for paused Kit tiled camera comparisons (0–255 space).""" +_KIT_PAUSED_TILED_CAMERA_NEWTON_CHANNEL_DIFF_THRESHOLD = 160 +"""Per-channel threshold for paused Kit tiled camera comparisons on Newton (0–255 space).""" + +_KIT_PAUSED_TILED_CAMERA_PHYSX_CHANNEL_DIFF_THRESHOLD = 80 +"""Per-channel threshold for paused Kit tiled camera comparisons on PhysX (0–255 space). + +Matches the viewport value but is kept separate: this cell measures 103 differing pixels at the +default threshold, so it needs its own floor rather than tracking whatever the viewport uses. +""" _FRAME_MIN_CHANNEL_RANGE = 10 """Minimum per-frame channel range to reject all-one-color images.""" @@ -580,13 +587,6 @@ def _assert_frames_remain_stable( ) -> None: """Assert two viewport frames are effectively unchanged while simulation is paused.""" n_diff = _count_significantly_differing_pixels(frame_a, frame_b, channel_diff_threshold=channel_diff_threshold) - logging.getLogger(__name__).info( - "%s pause stability: %d px differed at threshold %g (gate %d)", - case_label, - n_diff, - channel_diff_threshold, - max_differing_pixels, - ) assert n_diff <= max_differing_pixels, ( f"{case_label} failed to pause during {phase}: {n_diff} pixels differed, expected at most " f"{max_differing_pixels} with per-channel threshold {channel_diff_threshold} in 0-255 space. " @@ -1440,9 +1440,9 @@ def _attempt_pause(): phase="pausing", debug_phase="pausing_tiled", channel_diff_threshold=( - _KIT_PAUSED_TILED_CAMERA_CHANNEL_DIFF_THRESHOLD + _KIT_PAUSED_TILED_CAMERA_NEWTON_CHANNEL_DIFF_THRESHOLD if isinstance(visualizer, KitVisualizer) and physics_kind == "newton" - else _KIT_PAUSED_VIEWPORT_CHANNEL_DIFF_THRESHOLD + else _KIT_PAUSED_TILED_CAMERA_PHYSX_CHANNEL_DIFF_THRESHOLD if isinstance(visualizer, KitVisualizer) else _FRAME_MOTION_CHANNEL_DIFF_THRESHOLD ),