diff --git a/docs/index.rst b/docs/index.rst index 4b51d4974ccf..be4e8aa281bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -100,7 +100,9 @@ Table of Contents :caption: Concepts source/concepts/backends_and_presets + source/concepts/renderers source/concepts/actuators + source/concepts/sensors/index source/concepts/coupled_solvers source/concepts/scene_data_providers source/concepts/visualization diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css index 568c5d04da0d..3c00c4a7a7bc 100644 --- a/docs/source/_static/css/custom.css +++ b/docs/source/_static/css/custom.css @@ -183,6 +183,12 @@ figure.nurec-gif-contain img { white-space: nowrap; } +/* Allow long renderer preset names to wrap without changing their copied text. */ +.renderer-preset-grid figcaption span.pre { + overflow-wrap: anywhere; + white-space: normal; +} + /* Keep sphinx-copybutton visible on code blocks (not only on hover). */ button.copybtn { opacity: 1; diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-albedo.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-albedo.png new file mode 100644 index 000000000000..270550bcd771 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-albedo.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-depth.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-depth.png new file mode 100644 index 000000000000..02b31da62278 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-depth.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-instance-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-instance-segmentation.png new file mode 100644 index 000000000000..6b78e0ab0a24 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-instance-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-motion-vectors.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-motion-vectors.png new file mode 100644 index 000000000000..bf36638f465a Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-motion-vectors.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-normals.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-normals.png new file mode 100644 index 000000000000..d51ab11daba6 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-normals.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-semantic-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-semantic-segmentation.png new file mode 100644 index 000000000000..c76893c109c6 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-semantic-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-constant-diffuse.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-constant-diffuse.png new file mode 100644 index 000000000000..9a109a7b623c Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-constant-diffuse.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-diffuse-mdl.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-diffuse-mdl.png new file mode 100644 index 000000000000..24704c803711 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-diffuse-mdl.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-full-mdl.png b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-full-mdl.png new file mode 100644 index 000000000000..04a5bc1c87a7 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-full-mdl.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx.webp b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx.webp new file mode 100644 index 000000000000..f309c960b389 --- /dev/null +++ b/docs/source/_static/overview/sensors/camera-renderer-isaac-rtx.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4581c07c0480d2d5e6614b833a814515a2ff3aab335504acd6ba903c831d3b86 +size 428118 diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-albedo.png b/docs/source/_static/overview/sensors/camera-renderer-newton-albedo.png new file mode 100644 index 000000000000..ca96bf2143d1 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-albedo.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-depth.png b/docs/source/_static/overview/sensors/camera-renderer-newton-depth.png new file mode 100644 index 000000000000..8e14d787187d Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-depth.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-instance-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-newton-instance-segmentation.png new file mode 100644 index 000000000000..6aeebe40022c Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-instance-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-normals.png b/docs/source/_static/overview/sensors/camera-renderer-newton-normals.png new file mode 100644 index 000000000000..7f981aa59f58 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-normals.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-semantic-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-newton-semantic-segmentation.png new file mode 100644 index 000000000000..8485286f37f0 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-semantic-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-disabled.png b/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-disabled.png new file mode 100644 index 000000000000..a97c3281ea98 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-disabled.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-enabled.png b/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-enabled.png new file mode 100644 index 000000000000..c0694ac6b95b Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-newton-shadows-enabled.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-newton.webp b/docs/source/_static/overview/sensors/camera-renderer-newton.webp new file mode 100644 index 000000000000..51a92665492d --- /dev/null +++ b/docs/source/_static/overview/sensors/camera-renderer-newton.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1169a4ae6f8995a8f527466f27f9a6d4e62f56fd21e31a3b747ff17996aeb03d +size 756588 diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-albedo.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-albedo.png new file mode 100644 index 000000000000..c3d1b3cf3fd9 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-albedo.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-depth.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-depth.png new file mode 100644 index 000000000000..eaeee0b3ba66 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-depth.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-instance-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-instance-segmentation.png new file mode 100644 index 000000000000..8e92b39cfee8 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-instance-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-motion-vectors.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-motion-vectors.png new file mode 100644 index 000000000000..d27ac906962f Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-motion-vectors.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-normals.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-normals.png new file mode 100644 index 000000000000..d84445d0f75d Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-normals.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-semantic-segmentation.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-semantic-segmentation.png new file mode 100644 index 000000000000..756ae8c11365 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-semantic-segmentation.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-constant-diffuse.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-constant-diffuse.png new file mode 100644 index 000000000000..264a857f3b0b Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-constant-diffuse.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-diffuse-mdl.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-diffuse-mdl.png new file mode 100644 index 000000000000..0f9f054c27fb Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-diffuse-mdl.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-full-mdl.png b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-full-mdl.png new file mode 100644 index 000000000000..2f24f6a71856 Binary files /dev/null and b/docs/source/_static/overview/sensors/camera-renderer-ovrtx-simple-shading-full-mdl.png differ diff --git a/docs/source/_static/overview/sensors/camera-renderer-ovrtx.webp b/docs/source/_static/overview/sensors/camera-renderer-ovrtx.webp new file mode 100644 index 000000000000..852db9d0c092 --- /dev/null +++ b/docs/source/_static/overview/sensors/camera-renderer-ovrtx.webp @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e3880df57d9c8c6b09d652c3a27e0db9e59f9c661a742a44df00b1344a4910 +size 494132 diff --git a/docs/source/_static/overview/sensors/camera_depth.jpg b/docs/source/_static/overview/sensors/camera_depth.jpg deleted file mode 100644 index a53155c8653b..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_depth.jpg and /dev/null differ diff --git a/docs/source/_static/overview/sensors/camera_instance.jpg b/docs/source/_static/overview/sensors/camera_instance.jpg deleted file mode 100644 index 9715ba8c0746..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_instance.jpg and /dev/null differ diff --git a/docs/source/_static/overview/sensors/camera_instanceID.jpg b/docs/source/_static/overview/sensors/camera_instanceID.jpg deleted file mode 100644 index 985461567e6f..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_instanceID.jpg and /dev/null differ diff --git a/docs/source/_static/overview/sensors/camera_normals.jpg b/docs/source/_static/overview/sensors/camera_normals.jpg deleted file mode 100644 index cf78e454d166..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_normals.jpg and /dev/null differ diff --git a/docs/source/_static/overview/sensors/camera_rgb.jpg b/docs/source/_static/overview/sensors/camera_rgb.jpg deleted file mode 100644 index 826bc0ce9504..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_rgb.jpg and /dev/null differ diff --git a/docs/source/_static/overview/sensors/camera_semantic.jpg b/docs/source/_static/overview/sensors/camera_semantic.jpg deleted file mode 100644 index d15217ef9d4c..000000000000 Binary files a/docs/source/_static/overview/sensors/camera_semantic.jpg and /dev/null differ diff --git a/docs/source/concepts/_renderer_gallery.rst b/docs/source/concepts/_renderer_gallery.rst new file mode 100644 index 000000000000..26e14b3eef62 --- /dev/null +++ b/docs/source/concepts/_renderer_gallery.rst @@ -0,0 +1,223 @@ +.. tab-set:: + + .. tab-item:: Newton Warp + + .. figure:: ../_static/overview/sensors/camera-renderer-newton.webp + :align: center + :width: 90% + :alt: Six material spheres falling onto a table in Newton Warp RGB output. + + RGB animation + + .. grid:: 1 2 3 3 + :gutter: 2 + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-albedo.png + :width: 100% + :alt: Newton Warp albedo output for six material spheres. + + Albedo + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-depth.png + :width: 100% + :alt: Color-mapped Newton Warp depth output for six material spheres. + + Depth. Turbo colors progress from near (purple) to far (red). + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-normals.png + :width: 100% + :alt: Newton Warp surface-normal output for six material spheres. + + Normals + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-semantic-segmentation.png + :width: 100% + :alt: Color-mapped Newton Warp semantic-segmentation output for six material spheres. + + Semantic segmentation + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-instance-segmentation.png + :width: 100% + :alt: Color-mapped Newton Warp instance-segmentation output for six material spheres. + + Instance segmentation + + .. tab-item:: OVRTX + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx.webp + :align: center + :width: 90% + :alt: Six material spheres falling onto a table in OVRTX RGB output. + + RGB animation + + .. grid:: 1 2 3 3 + :gutter: 2 + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-albedo.png + :width: 100% + :alt: OVRTX albedo output for six material spheres. + + Albedo + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-depth.png + :width: 100% + :alt: Color-mapped OVRTX depth output for six material spheres. + + Depth. Turbo colors progress from near (purple) to far (red). + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-normals.png + :width: 100% + :alt: OVRTX surface-normal output for six material spheres. + + Normals + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-semantic-segmentation.png + :width: 100% + :alt: Color-mapped OVRTX semantic-segmentation output for six material spheres. + + Semantic segmentation + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-instance-segmentation.png + :width: 100% + :alt: Color-mapped OVRTX instance-segmentation output for six material spheres. + + Instance segmentation + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-motion-vectors.png + :width: 100% + :alt: Color-mapped OVRTX motion-vector output for six falling material spheres. + + Motion vectors. Color encodes direction and relative magnitude; arrows show sampled + image-space motion. + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-constant-diffuse.png + :width: 100% + :alt: OVRTX constant-diffuse simple-shading output for six material spheres. + + Constant diffuse shading + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-diffuse-mdl.png + :width: 100% + :alt: OVRTX diffuse-MDL simple-shading output for six material spheres. + + Diffuse MDL shading + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-full-mdl.png + :width: 100% + :alt: OVRTX full-MDL simple-shading output for six material spheres. + + Full MDL shading + + .. tab-item:: Isaac RTX + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx.webp + :align: center + :width: 90% + :alt: Six material spheres falling onto a table in Isaac RTX RGB output. + + RGB animation + + .. grid:: 1 2 3 3 + :gutter: 2 + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-albedo.png + :width: 100% + :alt: Isaac RTX albedo output for six material spheres. + + Albedo + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-depth.png + :width: 100% + :alt: Color-mapped Isaac RTX depth output for six material spheres. + + Depth. Turbo colors progress from near (purple) to far (red). + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-normals.png + :width: 100% + :alt: Isaac RTX surface-normal output for six material spheres. + + Normals + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-semantic-segmentation.png + :width: 100% + :alt: Color-mapped Isaac RTX semantic-segmentation output for six material spheres. + + Semantic segmentation + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-instance-segmentation.png + :width: 100% + :alt: Color-mapped Isaac RTX instance-segmentation output for six material spheres. + + Instance segmentation + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-motion-vectors.png + :width: 100% + :alt: Color-mapped Isaac RTX motion-vector output for six falling material spheres. + + Motion vectors. Color encodes direction and relative magnitude; arrows show sampled + image-space motion. + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-constant-diffuse.png + :width: 100% + :alt: Isaac RTX constant-diffuse simple-shading output for six material spheres. + + Constant diffuse shading + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-diffuse-mdl.png + :width: 100% + :alt: Isaac RTX diffuse-MDL simple-shading output for six material spheres. + + Diffuse MDL shading + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-full-mdl.png + :width: 100% + :alt: Isaac RTX full-MDL simple-shading output for six material spheres. + + Full MDL shading diff --git a/docs/source/concepts/backends_and_presets.rst b/docs/source/concepts/backends_and_presets.rst index 700efe6c5d50..163fc3df1e95 100644 --- a/docs/source/concepts/backends_and_presets.rst +++ b/docs/source/concepts/backends_and_presets.rst @@ -247,5 +247,5 @@ Where to go next factories, the physics manager, assets, and sensors dispatch across backends. - :doc:`/source/overview/core-concepts/physical-backends/index` compares physics backend capabilities and links to backend-specific setup guides. -- :doc:`/source/overview/core-concepts/renderers` explains renderer selection and +- :doc:`/source/concepts/renderers` explains renderer selection and implementation details. diff --git a/docs/source/concepts/renderers.rst b/docs/source/concepts/renderers.rst new file mode 100644 index 000000000000..71446da96da4 --- /dev/null +++ b/docs/source/concepts/renderers.rst @@ -0,0 +1,482 @@ +.. _concepts_renderers: +.. _overview_renderers: + +Renderers +========= + +Renderers produce camera-sensor buffers for policy observations and synthetic-data workflows. +They are distinct from :doc:`visualizers `, which provide +human-facing, interactive views for inspection, debugging, and recording. Isaac Lab uses a +pluggable renderer architecture. All implementations follow the interface defined by +:class:`~isaaclab.renderers.BaseRenderer`. + +Isaac Lab supports three rendering backends. The table below summarizes their runtime requirements +and primary use cases. See :ref:`renderer-details` for their execution models, output coverage, +and limitations, and :ref:`camera-supported-annotators` for the output-by-backend support matrix. + +Choosing a renderer backend +---------------------------- + ++---------------------+-------------------------------+---------------------------------+ +| Backend | Requires Isaac Sim? | Best For | ++=====================+===============================+=================================+ +| Isaac RTX | Yes | Full sensor fidelity, RTX | +| | | photorealism, PhysX backend | ++---------------------+-------------------------------+---------------------------------+ +| OVRTX | No (kit-less; needs | RTX-quality rendering without | +| | ``isaaclab_ov`` + ``ovrtx``) | requiring Isaac Sim | ++---------------------+-------------------------------+---------------------------------+ +| Newton Warp | No (kit-less) | Newton backend, fast training | ++---------------------+-------------------------------+---------------------------------+ + +.. _renderer-visual-comparison: + +Renderer outputs at a glance +---------------------------- + +The galleries below use the same authored scene, camera, lights, materials, and initial conditions. +Six spheres exercise mirror-like, transparent, semi-transparent, matte, glossy, and emissive +materials. The RGB output is animated to show the spheres falling onto the table; the remaining +outputs are still frames from the same run. Stills use the sixth rendered frame so temporal outputs, +such as motion vectors, show useful motion while the spheres remain near their initial poses. + +Treat the images as a qualitative comparison of feature coverage and image character, not a +performance benchmark. The kit-less renderers use Newton physics while Isaac RTX uses PhysX, so +the exact sphere poses can differ. Display-only color maps make scalar, vector, and label outputs +readable here; camera sensors still return their documented raw tensors. Closely related aliases +and distance or ID variants are omitted because they do not add a visually distinct mode. + +.. include:: _renderer_gallery.rst + +.. _renderer-details: + +Renderer details +---------------- + +Open a renderer below for its execution model, output coverage, limitations, and typical use case. +The gallery above remains a visual comparison; the camera support matrix is the authoritative +output-by-backend reference. + +.. dropdown:: Newton Warp renderer + + - **Runtime and physics:** A lightweight, kit-less rasterizer built on NVIDIA Warp and paired + with Newton physics. + - **Output coverage:** Produces RGB, albedo, depth, normals, semantic segmentation, and instance + segmentation. It does not produce motion vectors or RTX material transport. Its focused + raster pipeline exposes fewer ground-truth outputs than the RTX renderers, which integrate + broader RTX and annotator capabilities. + - **Best for:** Training workflows where throughput matters more than full RTX fidelity. + +.. dropdown:: OVRTX renderer + + - **Runtime and physics:** Provides kit-less RTX rendering through the ``isaaclab_ov`` extension + and pairs with Newton physics. + - **Output coverage:** Provides RTX Minimal and photo-real rendering with geometry, motion, and + label outputs. + - **Best for:** RTX image quality without running Isaac Sim. + +.. dropdown:: Isaac RTX renderer + + - **Runtime and physics:** Runs NVIDIA's Omniverse RTX rendering pipeline inside Isaac Sim and + pairs with PhysX. + - **Output coverage:** Provides RTX Minimal and photo-real rendering, plus the broadest + camera-output coverage. + - **Best for:** Full RTX fidelity and workflows that already depend on Isaac Sim or Kit. + +Choosing a rendering capability +-------------------------------- + +The overview above shows the complete range of visually distinct outputs in one place. The sections +below regroup those outputs by purpose: simplified rendering for throughput-oriented training, +photo-real rendering for full RTX image quality, and advanced outputs for geometry, motion, and +labels. The detailed captions and commands use the suffixless ``Isaac-Cartpole-Camera`` task so +each shown mode can be tried without editing Python. + +Simplified rendering +~~~~~~~~~~~~~~~~~~~~ + +Simplified rendering prioritizes throughput and predictable image formation over full light +transport. Newton Warp provides a lightweight rasterized RGB path. OVRTX and Isaac RTX provide RTX +Minimal mode, which disables indirect lighting and offers three levels of material evaluation: + +- **Constant diffuse** uses one constant surface color. +- **Diffuse MDL** is Isaac Lab's stable name for textured diffuse shading. +- **Full MDL** is Isaac Lab's stable name for diffuse, glossy, and emissive material evaluation. + +RTX Minimal uses the first distant light in the scene and hard shadows. See the upstream +`OVRTX Minimal mode `_ +and `RTX Minimal renderer `_ +documentation for the renderer-level settings and limitations. + +.. tab-set:: + + .. tab-item:: Newton Warp RGB + + .. grid:: 1 2 2 2 + :gutter: 2 + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-shadows-disabled.png + :width: 100% + :alt: Newton Warp RGB output with directional-light shadows disabled. + + Without shadows (default) + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-newton-shadows-enabled.png + :width: 100% + :alt: Newton Warp RGB output with directional-light shadows enabled. + + With shadows + + Newton Warp is the kit-less choice when a lightweight RGB observation is sufficient. + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl \ + --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=newton_renderer presets=rgb + + Enable directional-light shadows explicitly: + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl \ + --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=newton_renderer presets=rgb \ + env.scene.tiled_camera.renderer_cfg.enable_shadows=true + + .. tab-item:: OVRTX Minimal + + .. grid:: 1 2 3 3 + :gutter: 2 + :class-container: renderer-preset-grid + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-constant-diffuse.png + :width: 100% + :alt: OVRTX constant-diffuse RTX Minimal output. + + Constant diffuse — ``presets=simple_shading_constant_diffuse`` + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-diffuse-mdl.png + :width: 100% + :alt: OVRTX textured-diffuse RTX Minimal output. + + Diffuse MDL — ``presets=simple_shading_diffuse_mdl`` + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx-simple-shading-full-mdl.png + :width: 100% + :alt: OVRTX full-material RTX Minimal output. + + Full MDL — ``presets=simple_shading_full_mdl`` + + **Constant diffuse** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=ovrtx presets=simple_shading_constant_diffuse + + **Textured diffuse** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=ovrtx presets=simple_shading_diffuse_mdl + + **Diffuse, glossy, and emissive material evaluation** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=ovrtx presets=simple_shading_full_mdl + + .. tab-item:: Isaac RTX Minimal + + .. grid:: 1 2 3 3 + :gutter: 2 + :class-container: renderer-preset-grid + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-constant-diffuse.png + :width: 100% + :alt: Isaac RTX constant-diffuse RTX Minimal output. + + Constant diffuse — ``presets=simple_shading_constant_diffuse`` + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-diffuse-mdl.png + :width: 100% + :alt: Isaac RTX textured-diffuse RTX Minimal output. + + Diffuse MDL — ``presets=simple_shading_diffuse_mdl`` + + .. grid-item:: + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx-simple-shading-full-mdl.png + :width: 100% + :alt: Isaac RTX full-material RTX Minimal output. + + Full MDL — ``presets=simple_shading_full_mdl`` + + **Constant diffuse** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=isaacsim_physx renderer=isaacsim_rtx presets=simple_shading_constant_diffuse + + **Textured diffuse** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=isaacsim_physx renderer=isaacsim_rtx presets=simple_shading_diffuse_mdl + + **Diffuse, glossy, and emissive material evaluation** + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=isaacsim_physx renderer=isaacsim_rtx presets=simple_shading_full_mdl + +Photo-real rendering +~~~~~~~~~~~~~~~~~~~~ + +Here, **photo-real rendering** means the regular RGB path from the full RTX Real-Time Path-Tracing +mode; it is a capability grouping, not an Isaac Lab preset name. Choose it when material appearance, +reflections, transparency, lighting, or the accompanying RTX AOVs matter more than the lowest +possible render latency. OVRTX provides this path without Kit, while Isaac RTX provides it inside +Isaac Sim. See the upstream `OVRTX render modes +`_ and +`Isaac Sim rendering modes +`_. + +.. tab-set:: + + .. tab-item:: OVRTX + + .. figure:: ../_static/overview/sensors/camera-renderer-ovrtx.webp + :align: center + :width: 90% + :alt: Six material spheres falling onto a table in OVRTX RGB output. + + OVRTX photo-real RGB — ``renderer=ovrtx presets=rgb`` + + The same renderer also produces the albedo, depth, normals, segmentation, and motion-vector + outputs shown in the overview gallery. + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=newton_mjwarp renderer=ovrtx presets=rgb + + .. tab-item:: Isaac RTX + + .. figure:: ../_static/overview/sensors/camera-renderer-isaac-rtx.webp + :align: center + :width: 90% + :alt: Six material spheres falling onto a table in Isaac RTX RGB output. + + Isaac RTX photo-real RGB — ``renderer=isaacsim_rtx presets=rgb`` + + The same renderer also produces the albedo, depth, normals, segmentation, and motion-vector + outputs shown in the overview gallery. + + .. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl --task Isaac-Cartpole-Camera \ + physics=isaacsim_physx renderer=isaacsim_rtx presets=rgb + +Advanced rendering outputs +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Beyond RGB, renderer-produced buffers expose material, geometry, motion, and labeling information: + +- **Albedo** isolates the material base color from lighting. +- **Depth** measures optical-axis distance for geometric perception and reconstruction. +- **Normals** encode local surface orientation. +- **Semantic segmentation** groups pixels by class, while **instance segmentation** separates + individual objects. +- **Motion vectors** encode image-space motion per pixel and require prior-frame history. + +Request one or more buffers through :attr:`~sensors.CameraCfg.data_types`. See +:ref:`camera-configuration` for a configuration example and :ref:`camera-output-types` for the +available names, tensor shapes, data types, and meanings. Environment-level ``presets=...`` +selectors are task-defined convenience shortcuts, not an exhaustive list of camera outputs. + +Output availability differs by backend. Check the :ref:`camera renderer support matrix +` before choosing a renderer; for example, OVRTX and Isaac RTX produce +motion vectors, while Newton Warp does not. The :ref:`renderer visual comparison +` above shows these outputs on the same scene. + +.. note:: + + Visualization markers are debug overlays provided by visualizers, not camera outputs. Their + support is independent of the camera renderer: the Kit, Newton GL, Rerun, and Viser visualizers + support markers, while the experimental Newton RTX visualizer does not. See + :doc:`/source/concepts/visualization` for the visualizer support matrix. + +.. note:: + **Temporal information for camera-based RL.** Unlike RTX modes with temporal + anti-aliasing (DLSS, DLAA, TAA), the Newton Warp renderer does not inject + prior-frame information into the current image. Camera-control tasks that depend + on velocity-like visual cues should add explicit temporal observations + (e.g. task-local frame stacking) rather than relying on renderer-specific artifacts. + +Per-environment Isaac RTX scene partitioning +--------------------------------------------- + +The Isaac RTX renderer enables per-environment scene partitioning by default. It assigns +matching scene-partition tokens to each ``/World/envs/env_`` hierarchy and its +camera so tiled views render only that environment's geometry. + +Configure the behavior through :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`: + +.. code-block:: python + + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + renderer_cfg = IsaacRtxRendererCfg(enable_scene_partitioning=False) + +Scene partitioning and the all-environment spectator view are separate controls. +:class:`~isaaclab.app.AppLauncher` enables spectator support before RTX startup only +when the Kit viewport is enabled or Kit visualization, recording, livestreaming, or XR +is requested. Regular headless training and camera-sensor runs keep it disabled so +tiled cameras are not exposed to the spectator mode's world-space layout constraints. + +``global_settings.show_all_partitions_by_default`` maps to that same process-global RTX +setting; it is not a separate feature. Its default value of ``None`` preserves the +launch-time choice made by :class:`~isaaclab.app.AppLauncher`. An explicit value overrides +that setting when the Isaac RTX renderer is constructed. When enabled, environments must +remain spatially separated because overlapping partition bounds can make content leak into +another environment or disappear. When disabled, the Kit viewport displays only the +selected environment. + +This setting does not affect OVRTX, which always partitions multi-environment scenes. + +Prims outside the environment hierarchies remain in the shared background partition. +Environment-owned ``PointInstancer`` markers can carry one matching scene-partition +token per instance; markers without that ownership information remain shared. + +.. warning:: + + Kit RTX sizes each partition from the bounding boxes of the prims it contains and never + refreshes the bounding box of an animated ``UsdGeom.BasisCurves`` prim, so cables can be + culled once they deform beyond their initial extent. See + :ref:`known-issues-animated-curve-scene-partition` for the workaround. + +Architecture Overview +--------------------- + +The renderer system consists of: + +1. **BaseRenderer** — Abstract base class defining the rendering lifecycle and interface +2. **RendererCfg** — Base configuration; each backend extends it with backend-specific options and declares + its implementation in ``class_type`` +3. **Concrete implementations** — Backend-specific renderers in extension packages +4. **RenderContext** — A management class for instantiating and accessing renderer instances using a **RendererCfg**. + After instantiation, a config can then be used to acquire the instance of the renderer as needed. + +.. code-block:: python + + import isaaclab.sim as sim_utils + from isaaclab.renderers import BaseRenderer + from isaaclab_newton.renderers import NewtonWarpRendererCfg + + # Create a Newton Warp renderer (no Isaac Sim required) + sim_ctx = sim_utils.SimulationContext.instance() + # RenderContext.get_renderer constructs cfg.class_type(cfg) + # or return an existing renderer with a matching config + renderer: BaseRenderer = sim_ctx.render_context.get_renderer(NewtonWarpRendererCfg()) + assert isinstance(renderer, BaseRenderer) + +For the RTX renderer (requires Isaac Sim): + +.. code-block:: python + + import isaaclab.sim as sim_utils + from isaaclab.renderers import BaseRenderer + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + # Create an RTX renderer + sim_ctx = sim_utils.SimulationContext.instance() + # RenderContext.get_renderer constructs cfg.class_type(cfg) + # or return an existing renderer with a matching config + renderer: BaseRenderer = sim_ctx.render_context.get_renderer(IsaacRtxRendererCfg()) + +For RTX renderer settings, see +:doc:`/source/how-to/configure_rendering`. + +Core concepts +------------- + +- **Use the RenderContext**: Always acquire renderers via the RenderContext with a renderer-specific config class + (e.g. ``sim_ctx.render_context.get_renderer(IsaacRtxRendererCfg())``). Do not import or instantiate concrete backend classes + (e.g. ``IsaacRtxRenderer``, ``OVRTXRenderer``) directly—their names and package locations are + implementation details and may change without notice. + +- **Lightweight config imports**: Importing a renderer configuration class does not pull in backend-specific + dependencies. ``class_type`` is resolved lazily when the renderer is constructed, and construction may fail + if the backend is not installed. + + .. code-block:: python + + import isaaclab.sim as sim_utils + from isaaclab.renderers import BaseRenderer + # Lightweight: does not import OVRTX backend dependencies + from isaaclab_ov.renderers import OVRTXRendererCfg + + # Lazily loads ovrtx when instantiated; may fail if isaaclab_ov / ovrtx is not installed + sim_ctx = sim_utils.SimulationContext.instance() + renderer: BaseRenderer = sim_ctx.render_context.get_renderer(OVRTXRendererCfg()) + +Installing the OVRTX renderer +------------------------------ + +The OVRTX renderer is provided by the ``isaaclab_ov`` extension. The extension's +source package ships with the core install, but the renderer's ``ovrtx`` runtime +wheel (the `ovrtx `_ package, published +on public PyPI) is **not** installed by default. You must request it +explicitly — OVRTX does **not** require Isaac Sim. + +Install via the Isaac Lab CLI using the ``ov[ovrtx]`` token: + +.. code-block:: bash + + # Install the ovrtx runtime wheel on top of an existing install + ./isaaclab.sh -i ov[ovrtx] + +.. note:: + + The bare ``ov`` token does **not** install any runtime wheel (the source + packages are already part of the core install). Use ``ov[ovrtx]`` (or ``ov[all]``) + to pull in the ``ovrtx`` dependency. + +Or install the public ``ovrtx`` package directly from PyPI: + +.. isaaclab-ovrtx-install:: + +- **Opaque render data**: The render data object returned by :meth:`~isaaclab.renderers.BaseRenderer.create_render_data` is passed to + subsequent renderer methods. It should be completely opaque to the caller: inspecting or modifying it + via get/set attributes is an anti-pattern and breaks the API contract. + +.. note:: + + The :class:`~isaaclab.renderers.BaseRenderer` class is under active development and may change without notice. + +See Also +-------- + +- :doc:`/source/concepts/scene_data_providers`: how scene data flows from physics backends to renderers +- :doc:`/source/concepts/visualization` — lightweight visualizer backends for interactive feedback diff --git a/docs/source/concepts/scene_data_providers.rst b/docs/source/concepts/scene_data_providers.rst index 6b173009b104..0b4a5eeb5407 100644 --- a/docs/source/concepts/scene_data_providers.rst +++ b/docs/source/concepts/scene_data_providers.rst @@ -146,5 +146,5 @@ simulation-context construction time and is what triggers the shadow-model build See Also -------- -- :doc:`/source/overview/core-concepts/renderers`: renderer backends that consume scene data +- :doc:`/source/concepts/renderers`: renderer backends that consume scene data - :doc:`/source/concepts/visualization`: visualizer backends that consume scene data diff --git a/docs/source/concepts/sensors/camera.rst b/docs/source/concepts/sensors/camera.rst new file mode 100644 index 000000000000..20e423482fc0 --- /dev/null +++ b/docs/source/concepts/sensors/camera.rst @@ -0,0 +1,343 @@ +.. _concepts_sensors_camera: +.. _overview_sensors_camera: + +.. currentmodule:: isaaclab + +Camera +====== + +A :class:`~sensors.Camera` defines what to capture: camera pose, projection, resolution, sampling +period, and output data types. A renderer defines how those images are produced. Keeping these +responsibilities separate lets one camera configuration work with different physics and rendering +backends. + +Camera data is expensive compared with low-dimensional state. Isaac Lab therefore batches the +camera copies from cloned environments into tiled render passes and exposes the de-tiled result as +one device-resident buffer per requested output. + +Rendering model +--------------- + +:attr:`~sensors.CameraCfg.renderer_cfg` selects the renderer. A plain +:class:`~isaaclab.renderers.RendererCfg` requests the runtime default. Use a concrete configuration +when the renderer must be fixed: + +.. list-table:: + :header-rows: 1 + :widths: 34 23 43 + + * - Renderer configuration + - Requires Isaac Sim + - Characteristics + * - :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg` + - Yes + - Replicator and RTX rendering through Isaac Sim + * - :class:`~isaaclab_ov.renderers.OVRTXRendererCfg` + - No + - Kit-less RTX rendering through ``isaaclab_ov`` + * - :class:`~isaaclab_newton.renderers.NewtonWarpRendererCfg` + - No + - Kit-less Warp rasterization through Newton + +For an environment that exposes renderer presets, select the renderer at launch instead of editing +the scene configuration: + +.. code-block:: bash + + uv run isaaclab train --rl_library rsl_rl \ + --task Isaac-Cartpole-Camera-Direct renderer=newton_renderer + +See :doc:`/source/concepts/backends_and_presets` for preset discovery and +:ref:`renderer-visual-comparison` for a same-scene comparison of the renderer outputs. + +.. _camera-configuration: + +Configure a camera +------------------ + +A camera can spawn a pinhole or fisheye camera prim, or bind to a camera already on the stage. +``offset`` uses the convention declared on :class:`~sensors.CameraCfg.OffsetCfg`: + +* ``world``: forward ``+X``, up ``+Z``. +* ``ros``: forward ``+Z``, up ``-Y``. +* ``opengl``: forward ``-Z``, up ``+Y``. + +.. code-block:: python + + import isaaclab.sim as sim_utils + from isaaclab.sensors import CameraCfg + from isaaclab_newton.renderers import NewtonWarpRendererCfg + + front_camera = CameraCfg( + prim_path="{ENV_REGEX_NS}/Robot/base/front_camera", + update_period=0.05, + height=240, + width=320, + data_types=["rgb", "depth", "normals"], + spawn=sim_utils.PinholeCameraCfg( + focal_length=24.0, + horizontal_aperture=20.955, + clipping_range=(0.1, 20.0), + ), + offset=CameraCfg.OffsetCfg( + pos=(0.45, 0.0, 0.1), + rot=(0.0, 0.0, 0.0, 1.0), + convention="world", + ), + renderer_cfg=NewtonWarpRendererCfg(), + ) + +A renderer instance is reused only when cameras use equal renderer configurations of the same +concrete configuration type. Different renderer settings create distinct instances. Camera +configurations, including output types and backgrounds, remain per sensor. + +Read camera data +---------------- + +:attr:`~sensors.CameraData.output` maps each requested name to a +:class:`~isaaclab.utils.warp.ProxyArray`. For ``N`` camera views, height ``H``, width ``W``, and +``C`` channels, each output has shape ``(N, H, W, C)``. Use ``torch`` for a cached zero-copy Torch +view or ``warp`` for the underlying Warp array: + +.. code-block:: python + + camera_data = scene["front_camera"].data + rgb = camera_data.output["rgb"].torch + depth = camera_data.output["depth"].torch + intrinsics = camera_data.intrinsic_matrices.torch + +Camera pose and intrinsic buffers are also ``ProxyArray`` objects. ``pos_w`` has shape ``(N, 3)``, +``intrinsic_matrices`` has shape ``(N, 3, 3)``, and camera quaternions have shape ``(N, 4)`` in +``(x, y, z, w)`` order. Set ``update_latest_camera_pose=True`` only when current pose data is needed; +updating it adds frame-query overhead. + +.. _camera-output-types: + +Output types +------------ + +The active renderer validates ``data_types`` and allocates the channel count and data type declared +by its :class:`~isaaclab.renderers.RenderBufferSpec`. + +.. list-table:: Common output contracts + :header-rows: 1 + :widths: 34 22 44 + + * - Name + - Channels and type + - Meaning + * - ``rgb`` / ``rgba`` + - 3 / 4, ``uint8`` + - Low-dynamic-range color + * - ``rgb_hdr`` + - 3, ``float32`` + - Scene-linear high-dynamic-range color + * - ``albedo`` + - 4, ``uint8`` + - Material base color + * - ``depth`` / ``distance_to_image_plane`` + - 1, ``float32`` + - Distance [m] along the camera optical axis + * - ``distance_to_camera`` + - 1, ``float32`` + - Euclidean distance [m] from the optical center + * - ``normals`` + - 3, ``float32`` + - Local surface normal ``(x, y, z)`` + * - ``motion_vectors`` + - 2, ``float32`` + - Image-space motion; positive ``x`` is left and positive ``y`` is up + * - ``semantic_segmentation`` + - 4 ``uint8`` or 1 ``int32`` + - Semantic color or ID per pixel + * - ``instance_segmentation`` + - 4 ``uint8`` or 1 ``int32`` + - Semantically labeled instance color or ID per pixel + * - ``instance_id_segmentation_fast`` + - 4 ``uint8`` or 1 ``int32`` + - USD-prim instance color or ID per pixel + +``depth`` is an alias of ``distance_to_image_plane``. Colorized segmentation uses RGBA ``uint8``; +non-colorized segmentation uses one ``int32`` ID channel. Label and prim-path mappings are stored in +``camera_data.info[output_name]``. + +.. figure:: ../../_static/overview/sensors/camera-renderer-isaac-rtx.webp + :align: center + :figwidth: 100% + :alt: RGB camera output + + Isaac RTX RGB output. The animation shows the six material spheres falling onto the table. + +.. figure:: ../../_static/overview/sensors/camera-renderer-isaac-rtx-depth.png + :align: center + :figwidth: 100% + :alt: Depth camera output + + Isaac RTX depth output. Display colors encode optical-axis distance; the sensor returns metric + values [m]. + +.. _camera-supported-annotators: + +Renderer support +~~~~~~~~~~~~~~~~ + +The common API does not imply that every renderer produces every output. For same-scene examples +across these backends, see the :ref:`renderer visual comparison `. The +current support matrix is: + +.. list-table:: + :header-rows: 1 + :widths: 38 20 20 22 + + * - Output + - Isaac RTX + - OVRTX + - Newton Warp + * - ``rgb``, ``rgba``, ``rgb_hdr`` + - Yes + - Yes + - Yes + * - ``depth`` and both distance outputs + - Yes + - Yes + - Yes + * - ``normals`` + - Yes + - Yes + - Yes + * - ``albedo`` + - Isaac Sim 6.0+ + - Yes + - Yes + * - ``motion_vectors`` + - Yes + - Yes + - No + * - semantic and instance segmentation + - Yes + - Yes + - Yes + * - ``instance_id_segmentation_fast`` + - Yes + - No + - No + * - ``simple_shading_*`` modes + - Isaac Sim 6.0+ + - Yes + - No + +Querying an unsupported output fails during camera initialization. Renderer configuration controls +semantic filters, segmentation colorization, and depth clipping where those options are +backend-specific. + +.. figure:: ../../_static/overview/sensors/camera-renderer-isaac-rtx-normals.png + :align: center + :figwidth: 100% + :alt: Camera surface-normal output + + Isaac RTX normals output. Red, green, and blue encode the surface normal X, Y, and Z components. + +.. figure:: ../../_static/overview/sensors/camera-renderer-isaac-rtx-semantic-segmentation.png + :align: center + :figwidth: 100% + :alt: Semantic segmentation output + + Isaac RTX semantic segmentation. One color represents each class: the six spheres share one + class, while the table and backdrop use separate classes. + +.. figure:: ../../_static/overview/sensors/camera-renderer-isaac-rtx-instance-segmentation.png + :align: center + :figwidth: 100% + :alt: Instance segmentation output + + Isaac RTX instance segmentation. Each sphere receives its own color, distinguishing objects + that share the same semantic class. + +Background color +---------------- + +When :attr:`~sensors.CameraCfg.background_color` is ``None``, each renderer uses its default +background. Set a normalized RGB tuple to use a solid color for pixels that miss all geometry: + +.. code-block:: python + + mask_camera = front_camera.replace(background_color=(0.0, 0.0, 0.0)) + +The setting is per camera. Cameras with renderer-default and solid backgrounds can coexist in one +scene. + +Post-render image signal processing +----------------------------------- + +:attr:`~sensors.CameraCfg.isp_cfg` optionally applies an image signal processing (ISP) pass to the +renderer's scene-linear HDR output. The shipped implementation is PPISP (Physically Plausible Image +Signal Processing), which applies responsivity, exposure, vignetting, color correction, and a camera +response function before writing ``rgb`` or ``rgba``. + +The field accepts: + +* ``None`` to disable post-render ISP. +* :class:`~isaaclab_ppisp.PpispCfg` for explicit coefficients or coefficients imported from a USD + camera. +* :class:`~sensors.CameraISPMode` to discover ``ppisp:*`` attributes on a camera prim. + +.. code-block:: python + + from isaaclab.sensors.camera import CameraCfg, CameraISPMode + from isaaclab_ppisp import PpispCfg + + explicit_isp = front_camera.replace( + data_types=["rgb"], + isp_cfg=PpispCfg(inputs={"exposureOffset": 1.5}), + ) + + discovered_isp = front_camera.replace( + data_types=["rgb"], + isp_cfg=CameraISPMode.AUTO_CAMERA, + ) + +``AUTO_CAMERA`` checks the first matched camera prim. ``AUTO_ANY`` falls back to the first PPISP +camera anywhere on the stage. Discovery happens once during camera construction. + +PPISP is composed by Isaac RTX, OVRTX, and Newton Warp. It requires ``rgb`` or ``rgba`` output. A +static configuration is shared by all cloned views in one camera batch; controller weights may +predict per-view exposure and color parameters, while the remaining coefficients stay shared. ISP +configuration and discovered USD attributes are fixed for the camera lifetime. + +.. important:: + + With Isaac RTX and OVRTX, enabling ``isp_cfg`` makes PPISP the ISP authority. The renderer + disables RTX auto-exposure, authors neutral ``exposure:*`` values, and applies the + ``OmniRtxCameraAutoExposureAPI_1`` and ``OmniRtxCameraExposureAPI_1`` schemas on every matched + camera prim so RTX does not process the image a second time. Do not combine ``isp_cfg`` with + separately authored RTX exposure or tonemapping settings. When ``isp_cfg`` is ``None``, the + renderer leaves authored camera exposure unchanged. + +Run ``scripts/demos/sensors/ppisp_camera.py`` for a complete PPISP workflow: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/ppisp_camera.py \ + --renderer newton_renderer --max_steps 60 + +Performance and validation +-------------------------- + +Image memory and rendering cost scale with the number of environments, resolution, channel count, +and requested outputs. Start camera-based tasks with a small environment count, verify shapes and +renderer support, and then scale while monitoring GPU memory. Avoid requesting buffers that the task +does not consume. + +Tiled rendering batches the cloned views into shared render passes, but it does not remove the memory +cost of the de-tiled outputs or downstream vision models. The camera follows the shared sensor +``update_period`` contract; choose a period that matches the observation cadence instead of rendering +at every physics step by default. + +A runnable camera example is available in ``scripts/demos/sensors/cameras.py``: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/cameras.py + +For saving output to disk, see :doc:`/source/how-to/save_camera_output`. For camera-capacity +estimation, see :doc:`/source/how-to/estimate_how_many_cameras_can_run`. diff --git a/docs/source/concepts/sensors/contact_sensor.rst b/docs/source/concepts/sensors/contact_sensor.rst new file mode 100644 index 000000000000..d05b19e2e995 --- /dev/null +++ b/docs/source/concepts/sensors/contact_sensor.rst @@ -0,0 +1,145 @@ +.. _concepts_sensors_contact: +.. _overview_sensors_contact: + +.. currentmodule:: isaaclab + +Contact Sensor +============== + +A :class:`~sensors.ContactSensor` aggregates contacts on one or more rigid bodies into batched force +measurements. The sensor scope is defined by :attr:`~sensors.ContactSensorCfg.prim_path`: each matched +body becomes one sensor body in every environment. + +.. figure:: ../../_static/overview/sensors/contact_diagram.jpg + :align: center + :figwidth: 100% + :alt: A contact sensor reporting total and filtered contact forces + +Net and filtered forces +----------------------- + +:attr:`~sensors.ContactSensorData.net_normal_forces_w` is the aggregate normal force acting on each +sensor body in the world frame. It includes contact with every body in the scene. + +When supported by the backend and enabled with ``track_friction_forces``, +:attr:`~sensors.ContactSensorData.net_friction_forces_w` reports the aggregate friction force. +The total force is therefore + +.. math:: + + \boldsymbol{f}_{total} = \boldsymbol{f}_{normal} + \boldsymbol{f}_{friction}. + +On Newton, :attr:`~sensors.ContactSensorData.net_forces_w` reports this total. PhysX and OvPhysX +cannot compute the aggregate friction component, so ``net_forces_w`` returns +``net_normal_forces_w`` with a warning. Use the explicit normal and friction properties when the +split matters. + +Set :attr:`~sensors.ContactSensorCfg.filter_prim_paths_expr` when forces from specific collision +partners are also needed. :attr:`~sensors.ContactSensorData.normal_force_matrix_w` and +:attr:`~sensors.ContactSensorData.friction_force_matrix_w` retain one entry per configured +filter expression. + +Summing a force matrix over its filter dimension reconstructs the corresponding aggregate +force only when the filters cover every contacting object. + +Body-level filtering supports a many-to-one relationship: ``prim_path`` must resolve to one sensor +body per environment when filters are configured. Define one sensor per source body when separate +filtered forces are required, for example one sensor for each foot. Newton additionally supports +shape-level sensing and filtering through ``sensor_shape_prim_expr`` and +``filter_shape_prim_expr``. + +Configure the sensor +-------------------- + +Add the configuration to an :class:`~isaaclab.scene.InteractiveSceneCfg`: + +.. code-block:: python + + from isaaclab.scene import InteractiveSceneCfg + from isaaclab.sensors import ContactSensorCfg + + + class MySceneCfg(InteractiveSceneCfg): + left_foot_contact = ContactSensorCfg( + prim_path="{ENV_REGEX_NS}/Robot/LF_FOOT", + update_period=0.0, + history_length=6, + filter_prim_paths_expr=["{ENV_REGEX_NS}/Object"], + track_air_time=True, + ) + +``update_period=0.0`` samples every physics step. ``history_length`` stores earlier force samples. +Enable optional buffers only when they are needed. Their backend support differs: + +* Isaac Sim PhysX supports pose, filtered contact-point, and filtered friction-force + tracking, but not aggregate friction-force tracking. +* OvPhysX supports pose tracking for a single sensor body per environment, but not contact-point or + friction-force tracking. +* Newton supports filtered contact-point and aggregate or filtered friction-force tracking, but not + pose tracking. + +Filtered contact points and friction-force matrices require filters. Contact-rich Isaac +Sim PhysX scenes may require a larger ``max_contact_data_count_per_prim``. + +Read the data +------------- + +For ``E`` environments, ``S`` sensor bodies, ``F`` filter expressions, and history length ``H``, the +principal Torch views have these contracts: + +.. list-table:: + :header-rows: 1 + :widths: 32 26 42 + + * - Buffer + - Shape + - Meaning + * - ``net_normal_forces_w.torch`` + - ``(E, S, 3)`` + - Net normal contact force [N] in world frame + * - ``net_friction_forces_w.torch`` + - ``(E, S, 3)`` + - Net friction force [N] in world frame; Newton only + * - ``net_forces_w.torch`` + - ``(E, S, 3)`` + - Total force [N] on Newton; normal force with a warning on PhysX and OvPhysX + * - ``normal_force_matrix_w.torch`` + - ``(E, S, F, 3)`` + - Normal force [N] from each filtered partner + * - ``friction_force_matrix_w.torch`` + - ``(E, S, F, 3)`` + - Friction force [N] from each filtered partner; Isaac Sim PhysX and Newton + * - ``contact_pos_w.torch`` + - ``(E, S, F, 3)`` + - Average filtered contact position [m] in world frame; unavailable on OvPhysX + * - ``current_air_time.torch`` / ``current_contact_time.torch`` + - ``(E, S)`` + - Current mode duration [s] + +Supported optional buffers are ``None`` unless their matching tracking option or filter is enabled. +Normal and friction force histories follow the same shapes with an added ``H`` dimension. +OvPhysX rejects unsupported tracking options during initialization. Reading aggregate friction on +PhysX or OvPhysX raises ``NotImplementedError``. The compatibility alias ``friction_forces_w`` +returns the aggregate on Newton; on PhysX it returns ``friction_force_matrix_w`` with a warning. +Reading pose data on Newton raises ``NotImplementedError``. + +.. code-block:: python + + contact = scene["left_foot_contact"] + net_normal_force = contact.data.net_normal_forces_w.torch + object_normal_force = contact.data.normal_force_matrix_w.torch + +Use ``debug_vis=True`` while validating body expressions and filter partners. The visualization shows +sensor contacts but does not change the reported data. + +.. figure:: ../../_static/overview/sensors/contact_visualization.jpg + :align: center + :figwidth: 100% + :alt: Contact sensor debug visualization + +A complete runnable example is available in +``scripts/demos/sensors/contact_sensor.py``: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/contact_sensor.py diff --git a/docs/source/concepts/sensors/frame_transformer.rst b/docs/source/concepts/sensors/frame_transformer.rst new file mode 100644 index 000000000000..b8d597ea187c --- /dev/null +++ b/docs/source/concepts/sensors/frame_transformer.rst @@ -0,0 +1,74 @@ +.. _concepts_sensors_frame_transformer: +.. _overview_sensors_frame_transformer: + +.. currentmodule:: isaaclab + +Frame Transformer +================= + +A :class:`~sensors.FrameTransformer` tracks the pose of one or more target frames relative to a +source frame. It provides the same batched interface across cloned environments, avoiding repeated +USD traversal or per-environment transform calculations. + +.. figure:: ../../_static/overview/sensors/frame_transformer.jpg + :align: center + :figwidth: 100% + :alt: Source and target frames used by a frame transformer + +Define frames +------------- + +The sensor's :attr:`~sensors.FrameTransformerCfg.prim_path` selects the source rigid body. Each +:class:`~sensors.FrameTransformerCfg.FrameCfg` selects one or more target rigid bodies and can add a +fixed pose offset. Target paths accept regular expressions; the data order is recorded in +:attr:`~sensors.FrameTransformerData.target_frame_names`. + +.. code-block:: python + + from isaaclab.sensors import FrameTransformerCfg + + feet_in_base = FrameTransformerCfg( + prim_path="{ENV_REGEX_NS}/Robot/base", + target_frames=[ + FrameTransformerCfg.FrameCfg( + prim_path="{ENV_REGEX_NS}/Robot/.*_FOOT", + name="foot", + ), + ], + debug_vis=True, + ) + +A target expression is inclusive. If it also matches the source body, the output includes the +identity transform from the source to itself. Use a narrower expression when that entry is not +wanted. + +Read transforms +--------------- + +For ``E`` environments and ``T`` resolved target frames, positions have shape ``(E, T, 3)``, +quaternions have shape ``(E, T, 4)``, and combined poses have shape ``(E, T, 7)``. Quaternions use +``(x, y, z, w)`` order. + +.. code-block:: python + + transforms = scene["feet_in_base"].data + foot_names = transforms.target_frame_names + foot_pos_b = transforms.target_pos_source.torch + foot_quat_b = transforms.target_quat_source.torch + foot_pose_w = transforms.target_pose_w.torch + +``target_*_source`` fields express targets in the configured source frame. ``target_*_w`` fields +express the offset target frames in the world frame. The corresponding ``source_*_w`` fields expose +the offset source frame in world coordinates. Positions are in meters; quaternions are unitless. + +.. figure:: ../../_static/overview/sensors/frame_transformer_visualizer.jpg + :align: center + :figwidth: 100% + :alt: Frame transformer debug visualization + +A complete runnable example is available in +``scripts/demos/sensors/frame_transformer_sensor.py``: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/frame_transformer_sensor.py diff --git a/docs/source/concepts/sensors/imu.rst b/docs/source/concepts/sensors/imu.rst new file mode 100644 index 000000000000..6349ca26b72c --- /dev/null +++ b/docs/source/concepts/sensors/imu.rst @@ -0,0 +1,67 @@ +.. _concepts_sensors_imu: +.. _overview_sensors_imu: + +.. currentmodule:: isaaclab + +Inertial Measurement Unit (IMU) +=============================== + +A :class:`~sensors.Imu` models the two quantities produced by a conventional inertial measurement +unit: + +* :attr:`~sensors.ImuData.ang_vel_b`: angular velocity [rad/s] relative to the world, expressed in + the IMU frame. +* :attr:`~sensors.ImuData.lin_acc_b`: proper linear acceleration [m/s²], expressed in the IMU frame. + +Proper acceleration is what an accelerometer measures. It is zero in free fall and points upward +with magnitude :math:`g` for a stationary sensor supported against gravity. This differs from the +coordinate acceleration reported by the :doc:`pva`. + +.. figure:: ../../_static/overview/sensors/imu_diagram.jpg + :align: center + :figwidth: 100% + :alt: Forces that determine proper acceleration for an IMU + +Configure the sensor +-------------------- + +Attach the sensor to a rigid body or to a fixed child frame beneath one. ``offset`` places and +orients the measurement frame relative to the parent frame. + +.. code-block:: python + + from isaaclab.sensors import ImuCfg + + base_imu = ImuCfg( + prim_path="{ENV_REGEX_NS}/Robot/base/imu", + update_period=0.0, + offset=ImuCfg.OffsetCfg( + pos=(0.0, 0.0, 0.05), + rot=(0.0, 0.0, 0.0, 1.0), + ), + debug_vis=True, + ) + +Both data fields are :class:`~isaaclab.utils.warp.ProxyArray` buffers. For ``E`` environments, their +Torch views have shape ``(E, 3)``: + +.. code-block:: python + + imu_data = scene["base_imu"].data + angular_velocity = imu_data.ang_vel_b.torch + proper_acceleration = imu_data.lin_acc_b.torch + +The acceleration estimate depends on consecutive simulation states. Reset the scene and its sensors +together so derivative history is not carried across episodes. Use a sensor update period compatible +with the control loop that consumes the measurement. + +.. figure:: ../../_static/overview/sensors/imu_visualizer.jpg + :align: center + :figwidth: 100% + :alt: IMU acceleration debug visualization + +A complete runnable example is available in ``scripts/demos/sensors/imu_sensor.py``: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/imu_sensor.py diff --git a/docs/source/concepts/sensors/index.rst b/docs/source/concepts/sensors/index.rst new file mode 100644 index 000000000000..fe48c7b69810 --- /dev/null +++ b/docs/source/concepts/sensors/index.rst @@ -0,0 +1,71 @@ +.. _concepts_sensors: +.. _overview_sensors: + +Sensors +======== + +.. seealso:: + + These pages are the source of truth for the ``isaaclab-using-sensors-actuators`` agent skill + (`skills/user/use-sensors-actuators/ <../../../../skills/user/use-sensors-actuators/SKILL.md>`__). + When sensor behavior changes, keep the API documentation, maintained demos, these concept pages, + and the skill synchronized. + +Sensors turn simulation state or rendered scene data into batched measurements. Every Isaac Lab +sensor derives from :class:`~isaaclab.sensors.SensorBase` and follows the same lifecycle: + +* ``prim_path`` selects the prim or prims measured in every cloned environment. +* ``update_period`` sets the sampling period in simulated seconds. A value of ``0.0`` samples every + simulation step. +* :meth:`~isaaclab.sensors.SensorBase.update` advances the sensor clock. Sensor data is evaluated + lazily when the :attr:`~isaaclab.sensors.SensorBase.data` property is read, unless recomputation or + debug visualization is requested. +* :meth:`~isaaclab.sensors.SensorBase.reset` clears per-environment timestamps and internal state. + +Sensor data is exposed through :class:`~isaaclab.utils.warp.ProxyArray` buffers, including camera +outputs. Use the ``torch`` property for a cached zero-copy Torch view or ``warp`` for the underlying +Warp array. + +Choose a sensor +--------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 47 25 + + * - Sensor + - Measurement + - Typical use + * - :doc:`camera` + - Renderer-produced color, depth, normals, motion, and segmentation images + - Vision observations and synthetic data + * - :doc:`ray_caster` + - First collision locations for configurable ray patterns + - Terrain sensing, lidar-style queries, and ray-cast cameras + * - :doc:`contact_sensor` + - Net and filtered contact forces, contact points, friction, and contact timing + - Locomotion, grasping, and collision events + * - :doc:`frame_transformer` + - Relative and world poses for configured frames + - End-effector, foot, and object tracking + * - :doc:`imu` + - Angular velocity and proper linear acceleration + - Inertial observations + * - :doc:`pva` + - Ground-truth pose, velocity, coordinate acceleration, and projected gravity + - State estimation and privileged observations + * - :doc:`joint_wrench_sensor` + - Incoming joint reaction force and torque + - Force/torque sensing and contact-rich control + +.. toctree:: + :maxdepth: 1 + :hidden: + + camera + ray_caster + contact_sensor + frame_transformer + imu + pva + joint_wrench_sensor diff --git a/docs/source/concepts/sensors/joint_wrench_sensor.rst b/docs/source/concepts/sensors/joint_wrench_sensor.rst new file mode 100644 index 000000000000..9b2c84511c81 --- /dev/null +++ b/docs/source/concepts/sensors/joint_wrench_sensor.rst @@ -0,0 +1,62 @@ +.. _concepts_sensors_joint_wrench: +.. _overview_sensors_joint_wrench: + +.. currentmodule:: isaaclab + +Joint Wrench Sensor +=================== + +A :class:`~sensors.JointWrenchSensor` reports the incoming reaction wrench at each selected +articulation body's parent joint. It exposes force [N] and torque [N·m] separately, with entries +ordered by :attr:`~sensors.JointWrenchSensor.body_names`. + +Wrench convention +----------------- + +The ``incoming_joint_frame`` convention expresses the wrench in the child-side joint frame at the +child-side joint anchor. This matches the placement of a six-axis force/torque sensor mounted at the +joint. Backend implementations convert their native solver output to this common convention. + +Configure the sensor +-------------------- + +Set :attr:`~sensors.JointWrenchSensorCfg.prim_path` to the articulation root. Reported body coverage +depends on the physics backend: + +* PhysX and OVPhysX report every articulation link, including the root link. +* Newton reports the child link of each non-free, non-fixed joint. It therefore excludes the root + link and any links connected through free or fixed joints. + +Use :attr:`~sensors.JointWrenchSensor.body_names` or +:meth:`~sensors.JointWrenchSensor.find_bodies` instead of assuming that different backends expose +the same number or order of entries: + +.. literalinclude:: ../../../../source/isaaclab_tasks/isaaclab_tasks/core/locomotion/ant/ant_manager_env_cfg.py + :language: python + :lines: 78-82 + +Manager-based environments can select a body subset through +:class:`~isaaclab.managers.SceneEntityCfg` and use +:func:`~isaaclab.envs.mdp.body_incoming_wrench` as an observation term: + +.. literalinclude:: ../../../../source/isaaclab_tasks/isaaclab_tasks/core/locomotion/ant/ant_manager_env_cfg.py + :language: python + :lines: 122-131 + +Read the data +------------- + +For ``E`` environments and ``B`` reported bodies, ``force.torch`` and ``torque.torch`` each have +shape ``(E, B, 3)``. Both buffers are ``None`` before simulation initialization. + +.. code-block:: python + + joint_wrench = scene["joint_wrench"] + foot_ids, _ = joint_wrench.find_bodies([".*foot"]) + + force = joint_wrench.data.force.torch[:, foot_ids] + torque = joint_wrench.data.torque.torch[:, foot_ids] + wrench = torch.cat((force, torque), dim=-1) + +The composed ``wrench`` has shape ``(E, num_selected_bodies, 6)`` with force components followed by +torque components. ``B`` and the entry ordering can differ between backends. diff --git a/docs/source/concepts/sensors/pva.rst b/docs/source/concepts/sensors/pva.rst new file mode 100644 index 000000000000..143a63159f42 --- /dev/null +++ b/docs/source/concepts/sensors/pva.rst @@ -0,0 +1,79 @@ +.. _concepts_sensors_pva: +.. _overview_sensors_pva: + +.. currentmodule:: isaaclab + +Pose Velocity Acceleration (PVA) Sensor +======================================= + +A :class:`~sensors.Pva` reads the ground-truth kinematic state of a frame. Use it for privileged +observations, state estimation, or control inputs that require more than an inertial sensor measures. +Use an :doc:`imu` when the observation should follow accelerometer and gyroscope conventions. + +The sensor can attach directly to a rigid body or to a fixed child prim beneath a rigid-body +ancestor. In the latter case, Isaac Lab composes the child's fixed transform with the configured +sensor offset. + +Measurement contract +-------------------- + +For ``E`` environments, every vector has shape ``(E, 3)``. The pose has shape ``(E, 7)`` in +``(x, y, z, qx, qy, qz, qw)`` order. + +.. list-table:: + :header-rows: 1 + :widths: 28 27 45 + + * - Buffer + - Frame and units + - Meaning + * - ``pose_w`` + - World; [m, unitless] + - Sensor position and orientation + * - ``projected_gravity_b`` + - Sensor; unitless + - Unit gravity direction projected into the sensor frame + * - ``lin_vel_b`` + - Sensor; [m/s] + - Linear velocity relative to the world + * - ``ang_vel_b`` + - Sensor; [rad/s] + - Angular velocity relative to the world + * - ``lin_acc_b`` + - Sensor; [m/s²] + - Coordinate linear acceleration; zero at rest and :math:`-g` in free fall + * - ``ang_acc_b`` + - Sensor; [rad/s²] + - Angular acceleration relative to the world + +Configure and read the sensor +----------------------------- + +.. code-block:: python + + from isaaclab.sensors import PvaCfg + + base_state = PvaCfg( + prim_path="{ENV_REGEX_NS}/Robot/base", + update_period=0.0, + offset=PvaCfg.OffsetCfg(pos=(0.0, 0.0, 0.05)), + ) + +The data fields are :class:`~isaaclab.utils.warp.ProxyArray` buffers. Convert them to Torch views only +where Torch operations are required: + +.. code-block:: python + + pva_data = scene["base_state"].data + pose_w = pva_data.pose_w.torch + linear_velocity = pva_data.lin_vel_b.torch + coordinate_acceleration = pva_data.lin_acc_b.torch + +Like the IMU, acceleration uses state from consecutive simulation updates. Reset the scene and sensor +state together at episode boundaries. + +A complete runnable example is available in ``scripts/demos/sensors/pva_sensor.py``: + +.. code-block:: bash + + uv run --extra isaacsim python scripts/demos/sensors/pva_sensor.py diff --git a/docs/source/overview/core-concepts/sensors/ray_caster.rst b/docs/source/concepts/sensors/ray_caster.rst similarity index 93% rename from docs/source/overview/core-concepts/sensors/ray_caster.rst rename to docs/source/concepts/sensors/ray_caster.rst index a9fdf1036e44..da3068791602 100644 --- a/docs/source/overview/core-concepts/sensors/ray_caster.rst +++ b/docs/source/concepts/sensors/ray_caster.rst @@ -1,3 +1,4 @@ +.. _concepts_sensors_ray_caster: .. _overview_sensors_ray_caster: .. currentmodule:: isaaclab @@ -5,7 +6,7 @@ Ray Caster ============= -.. figure:: ../../../_static/overview/sensors/raycaster_patterns.jpg +.. figure:: ../../_static/overview/sensors/raycaster_patterns.jpg :align: center :figwidth: 100% :alt: A diagram outlining the basic geometry of frame transformations @@ -23,13 +24,13 @@ explicit and to access Newton-specific options. Using a ray caster sensor requires a **pattern** and a parent xform to be attached to. The pattern defines how the rays are cast, while the prim properties defines the orientation and position of the sensor (additional offsets can be specified for more exact placement). Isaac Lab supports a number of ray casting pattern configurations, including a generic LIDAR and grid pattern. -.. literalinclude:: ../../../../../scripts/demos/sensors/raycaster_sensor.py +.. literalinclude:: ../../../../scripts/demos/sensors/raycaster_sensor.py :language: python :lines: 40-71 Notice that the units on the pattern config is in degrees! Also, we enable visualization here to explicitly show the pattern in the rendering, but this is not required and should be disabled for performance tuning. -.. figure:: ../../../_static/overview/sensors/raycaster_visualizer.jpg +.. figure:: ../../_static/overview/sensors/raycaster_visualizer.jpg :align: center :figwidth: 100% :alt: Lidar Pattern visualized @@ -79,6 +80,6 @@ You can use this script to experiment with pattern configurations and build an i .. dropdown:: Code for raycaster_sensor.py :icon: code - .. literalinclude:: ../../../../../scripts/demos/sensors/raycaster_sensor.py + .. literalinclude:: ../../../../scripts/demos/sensors/raycaster_sensor.py :language: python :linenos: diff --git a/docs/source/concepts/visualization.rst b/docs/source/concepts/visualization.rst index 0add281a9f2c..2f6227c385b2 100644 --- a/docs/source/concepts/visualization.rst +++ b/docs/source/concepts/visualization.rst @@ -927,7 +927,7 @@ See Also - :doc:`/source/features/record_video`: recording MP4 clips from a visualizer or sensor - :doc:`/source/features/draw_markers`: creating and configuring custom visualization markers - :doc:`/source/how-to/capture_sensor_frames`: saving per-frame sensor outputs during training -- :doc:`/source/overview/core-concepts/renderers`: renderer backends (RTX, Newton Warp, OVRTX) +- :doc:`/source/concepts/renderers`: renderer backends (RTX, Newton Warp, OVRTX) - :doc:`/source/concepts/scene_data_providers`: how scene data flows to visualizers - :doc:`/source/overview/core-concepts/physical-backends/newton/index`: Newton backend guide - :doc:`/source/migration/migrating_to_isaaclab_3-0`: visualizer migration reference diff --git a/docs/source/how-to/capture_sensor_frames.rst b/docs/source/how-to/capture_sensor_frames.rst index fcc64007fcbd..97ea7511ae00 100644 --- a/docs/source/how-to/capture_sensor_frames.rst +++ b/docs/source/how-to/capture_sensor_frames.rst @@ -217,5 +217,5 @@ See also * :doc:`/source/concepts/reinforcement_learning` - the shared ``train``/``play`` entrypoints that register the capture flags * :doc:`/source/concepts/visualization` - visualizers and rendering during training -* :doc:`/source/overview/core-concepts/sensors/camera` - camera sensors and annotator data types +* :doc:`/source/concepts/sensors/camera` - camera sensors and annotator data types * :doc:`proxy_array` - dual CPU/GPU sensor buffers read through ``.torch`` diff --git a/docs/source/overview/core-concepts/index.rst b/docs/source/overview/core-concepts/index.rst index 5d6eb1eb019e..f0ad7b344614 100644 --- a/docs/source/overview/core-concepts/index.rst +++ b/docs/source/overview/core-concepts/index.rst @@ -12,6 +12,4 @@ This section we introduce core concepts in Isaac Lab. schema_cfgs schema_fragments task_workflows - sensors/index.rst - renderers motion_generators diff --git a/docs/source/overview/core-concepts/multi_backend_architecture.rst b/docs/source/overview/core-concepts/multi_backend_architecture.rst index c3f278c421d7..58690252b7c3 100644 --- a/docs/source/overview/core-concepts/multi_backend_architecture.rst +++ b/docs/source/overview/core-concepts/multi_backend_architecture.rst @@ -474,4 +474,4 @@ See Also - :doc:`physical-backends/newton/index` — Newton backend guide - :doc:`physical-backends/newton/newton-manager-abstraction` — adding Newton solver managers and coupled solvers -- :doc:`renderers` — renderer backend architecture +- :doc:`/source/concepts/renderers` — renderer backend architecture diff --git a/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst b/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst index 5e39b1f37b0f..c2b513dac330 100644 --- a/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst +++ b/docs/source/overview/core-concepts/physical-backends/newton/supported-features.rst @@ -73,7 +73,7 @@ The following sensors are backend-agnostic (implemented in ``isaaclab`` core) and work transparently with Newton: * Ray Caster -* Camera — see :doc:`../../sensors/camera` +* Camera — see :doc:`/source/concepts/sensors/camera` isaaclab_assets ^^^^^^^^^^^^^^^ diff --git a/docs/source/overview/core-concepts/physical-backends/physx/supported-features.rst b/docs/source/overview/core-concepts/physical-backends/physx/supported-features.rst index f94293f2d91b..cf14a09ce883 100644 --- a/docs/source/overview/core-concepts/physical-backends/physx/supported-features.rst +++ b/docs/source/overview/core-concepts/physical-backends/physx/supported-features.rst @@ -39,7 +39,7 @@ The following sensors are backend-agnostic (implemented in ``isaaclab`` core) and work transparently with PhysX: * Ray Caster -* Camera — see :doc:`../../sensors/camera` +* Camera — see :doc:`/source/concepts/sensors/camera` Rendering diff --git a/docs/source/overview/core-concepts/renderers.rst b/docs/source/overview/core-concepts/renderers.rst deleted file mode 100644 index 942b206fba93..000000000000 --- a/docs/source/overview/core-concepts/renderers.rst +++ /dev/null @@ -1,194 +0,0 @@ -.. _overview_renderers: - -Renderers -========= - -Isaac Lab uses a pluggable renderer architecture to support different rendering backends for camera sensors. -The :class:`~isaaclab.renderers.BaseRenderer` abstract base class defines the interface that all renderer -implementations must follow. - -Isaac Lab supports three rendering backends: - -- **Isaac RTX renderer** (``IsaacRtxRendererCfg``) — NVIDIA's Omniverse RTX rendering pipeline - running inside Isaac Sim. Requires Isaac Sim. Best for photorealistic rendering, full camera - sensor support (RGB, depth, semantic segmentation, etc.), and production quality outputs. -- **OVRTX renderer** (``OVRTXRendererCfg``) — A standalone RTX path-tracing renderer provided by - the ``isaaclab_ov`` extension. Delivers RTX-quality rendering. -- **Newton Warp renderer** (``NewtonWarpRendererCfg``) — A lightweight GPU-accelerated renderer - built on NVIDIA Warp. Works with the Newton physics backend and does **not** require Isaac Sim - (kit-less mode). Ideal for training workflows where full RTX fidelity is not needed. - -Choosing a renderer backend ----------------------------- - -+---------------------+-------------------------------+---------------------------------+ -| Backend | Requires Isaac Sim? | Best For | -+=====================+===============================+=================================+ -| Isaac RTX | Yes | Full sensor fidelity, RTX | -| | | photorealism, PhysX backend | -+---------------------+-------------------------------+---------------------------------+ -| OVRTX | No (kit-less; needs | RTX-quality rendering without | -| | ``isaaclab_ov`` + ``ovrtx``) | requiring Isaac Sim | -+---------------------+-------------------------------+---------------------------------+ -| Newton Warp | No (kit-less) | Newton backend, fast training | -+---------------------+-------------------------------+---------------------------------+ - -.. note:: - - Visualization markers are not yet supported by Newton-based renderer backends, - including the Newton Warp renderer. Use an RTX-based renderer, such as the - Isaac RTX renderer or OVRTX renderer, when marker visualization is needed. - -.. note:: - **Temporal information for camera-based RL.** Unlike RTX modes with temporal - anti-aliasing (DLSS, DLAA, TAA), the Newton Warp renderer does not inject - prior-frame information into the current image. Camera-control tasks that depend - on velocity-like visual cues should add explicit temporal observations - (e.g. task-local frame stacking) rather than relying on renderer-specific artifacts. - -Per-environment Isaac RTX scene partitioning ---------------------------------------------- - -The Isaac RTX renderer enables per-environment scene partitioning by default. It assigns -matching scene-partition tokens to each ``/World/envs/env_`` hierarchy and its -camera so tiled views render only that environment's geometry. - -Configure the behavior through :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`: - -.. code-block:: python - - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - renderer_cfg = IsaacRtxRendererCfg(enable_scene_partitioning=False) - -Scene partitioning and the all-environment spectator view are separate controls. -:class:`~isaaclab.app.AppLauncher` enables spectator support before RTX startup only -when the Kit viewport is enabled or Kit visualization, recording, livestreaming, or XR -is requested. Regular headless training and camera-sensor runs keep it disabled so -tiled cameras are not exposed to the spectator mode's world-space layout constraints. - -``global_settings.show_all_partitions_by_default`` maps to that same process-global RTX -setting; it is not a separate feature. Its default value of ``None`` preserves the -launch-time choice made by :class:`~isaaclab.app.AppLauncher`. An explicit value overrides -that setting when the Isaac RTX renderer is constructed. When enabled, environments must -remain spatially separated because overlapping partition bounds can make content leak into -another environment or disappear. When disabled, the Kit viewport displays only the -selected environment. - -This setting does not affect OVRTX, which always partitions multi-environment scenes. - -Prims outside the environment hierarchies remain in the shared background partition. -Environment-owned ``PointInstancer`` markers can carry one matching scene-partition -token per instance; markers without that ownership information remain shared. - -.. warning:: - - Kit RTX sizes each partition from the bounding boxes of the prims it contains and never - refreshes the bounding box of an animated ``UsdGeom.BasisCurves`` prim, so cables can be - culled once they deform beyond their initial extent. See - :ref:`known-issues-animated-curve-scene-partition` for the workaround. - -Architecture Overview ---------------------- - -The renderer system consists of: - -1. **BaseRenderer** — Abstract base class defining the rendering lifecycle and interface -2. **RendererCfg** — Base configuration; each backend extends it with backend-specific options and declares - its implementation in ``class_type`` -3. **Concrete implementations** — Backend-specific renderers in extension packages -4. **RenderContext** — A management class for instantiating and accessing renderer instances using a **RendererCfg**. - After instantiation, a config can then be used to acquire the instance of the renderer as needed. - -.. code-block:: python - - import isaaclab.sim as sim_utils - from isaaclab.renderers import BaseRenderer - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - # Create a Newton Warp renderer (no Isaac Sim required) - sim_ctx = sim_utils.SimulationContext.instance() - # RenderContext.get_renderer constructs cfg.class_type(cfg) - # or return an existing renderer with a matching config - renderer: BaseRenderer = sim_ctx.render_context.get_renderer(NewtonWarpRendererCfg()) - assert isinstance(renderer, BaseRenderer) - -For the RTX renderer (requires Isaac Sim): - -.. code-block:: python - - import isaaclab.sim as sim_utils - from isaaclab.renderers import BaseRenderer - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - # Create an RTX renderer - sim_ctx = sim_utils.SimulationContext.instance() - # RenderContext.get_renderer constructs cfg.class_type(cfg) - # or return an existing renderer with a matching config - renderer: BaseRenderer = sim_ctx.render_context.get_renderer(IsaacRtxRendererCfg()) - -For RTX renderer settings, see -:doc:`/source/how-to/configure_rendering`. - -Core concepts -------------- - -- **Use the RenderContext**: Always acquire renderers via the RenderContext with a renderer-specific config class - (e.g. ``sim_ctx.render_context.get_renderer(IsaacRtxRendererCfg())``). Do not import or instantiate concrete backend classes - (e.g. ``IsaacRtxRenderer``, ``OVRTXRenderer``) directly—their names and package locations are - implementation details and may change without notice. - -- **Lightweight config imports**: Importing a renderer configuration class does not pull in backend-specific - dependencies. ``class_type`` is resolved lazily when the renderer is constructed, and construction may fail - if the backend is not installed. - - .. code-block:: python - - import isaaclab.sim as sim_utils - from isaaclab.renderers import BaseRenderer - # Lightweight: does not import OVRTX backend dependencies - from isaaclab_ov.renderers import OVRTXRendererCfg - - # Lazily loads ovrtx when instantiated; may fail if isaaclab_ov / ovrtx is not installed - sim_ctx = sim_utils.SimulationContext.instance() - renderer: BaseRenderer = sim_ctx.render_context.get_renderer(OVRTXRendererCfg()) - -Installing the OVRTX renderer ------------------------------- - -The OVRTX renderer is provided by the ``isaaclab_ov`` extension. The extension's -source package ships with the core install, but the renderer's ``ovrtx`` runtime -wheel (the `ovrtx `_ package, published -on public PyPI) is **not** installed by default. You must request it -explicitly — OVRTX does **not** require Isaac Sim. - -Install via the Isaac Lab CLI using the ``ov[ovrtx]`` token: - -.. code-block:: bash - - # Install the ovrtx runtime wheel on top of an existing install - ./isaaclab.sh -i ov[ovrtx] - -.. note:: - - The bare ``ov`` token does **not** install any runtime wheel (the source - packages are already part of the core install). Use ``ov[ovrtx]`` (or ``ov[all]``) - to pull in the ``ovrtx`` dependency. - -Or install the public ``ovrtx`` package directly from PyPI: - -.. isaaclab-ovrtx-install:: - -- **Opaque render data**: The render data object returned by :meth:`~isaaclab.renderers.BaseRenderer.create_render_data` is passed to - subsequent renderer methods. It should be completely opaque to the caller: inspecting or modifying it - via get/set attributes is an anti-pattern and breaks the API contract. - -.. note:: - - The :class:`~isaaclab.renderers.BaseRenderer` class is under active development and may change without notice. - -See Also --------- - -- :doc:`/source/concepts/scene_data_providers`: how scene data flows from physics backends to renderers -- :doc:`/source/concepts/visualization` — lightweight visualizer backends for interactive feedback diff --git a/docs/source/overview/core-concepts/sensors/camera.rst b/docs/source/overview/core-concepts/sensors/camera.rst deleted file mode 100644 index 2986394f8d4b..000000000000 --- a/docs/source/overview/core-concepts/sensors/camera.rst +++ /dev/null @@ -1,579 +0,0 @@ -.. _overview_sensors_camera: - -.. currentmodule:: isaaclab - -Camera -====== - -Camera sensors in Isaac Lab are renderer-backed sensors: each :class:`~sensors.Camera` instance -is coupled to a **renderer** that produces the image data. If multiple cameras use the same renderer -type, only one renderer is instantiated and shared between them. The renderer and camera are intentionally -isolated from each other — the camera defines *what* to capture (pose, resolution, field of view, -data types), while the renderer defines *how* to render it (RTX ray-tracing, Newton Warp rasterizer, -etc.). This separation allows the same camera configuration to run across different physics and -rendering backends without code changes. - -For an overview of the available renderer backends and how to choose between them, see -:ref:`overview_renderers`. - -Rendered images are unique among supported sensor data types due to their large bandwidth requirements. -A single 800 × 600 image with 32-bit color clocks in at just under 2 MB. At 60 fps across thousands -of parallel environments, this grows quickly. Isaac Lab's tiled rendering API specifically addresses -these scaling challenges by batching all cameras into a single render pass. - - -Renderer Backends ------------------ - -The renderer used by a camera is configured via the ``renderer_cfg`` field on -:class:`~sensors.CameraCfg`. The default is :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg` -(NVIDIA RTX, requires Isaac Sim). - -.. list-table:: - :header-rows: 1 - :widths: 30 30 - - * - ``renderer_cfg`` - - Requires Isaac Sim? - * - ``IsaacRtxRendererCfg`` *(default)* - - Yes - * - ``NewtonWarpRendererCfg`` - - No (kit-less) - * - ``OVRTXRendererCfg`` - - No (+ ``isaaclab_ov``) - -.. note:: - - Backends differ in which annotators they produce. See - :ref:`the support matrix ` below for the - per-annotator breakdown across the Isaac RTX, OVRTX, and Newton Warp renderers. - - -Tiled Rendering -~~~~~~~~~~~~~~~ - -.. note:: - - This feature is available from Isaac Sim version 4.2.0 onwards (for the RTX renderer). - The Newton Warp renderer supports tiled rendering in kit-less mode. - - Tiled rendering in combination with image processing networks require heavy memory resources, - especially at larger resolutions. We recommend running 512 cameras on RTX 4090 GPUs or similar - when using the RTX renderer. - -The Tiled Rendering API provides a vectorized interface for collecting image data from all environment -clones in a single batched render pass. Instead of one render call per camera, all copies of a camera -are composited into a single large tiled image, dramatically reducing host-device transfer overhead. - -Isaac Lab provides tiled rendering through :class:`~sensors.Camera`, configured via -:class:`~sensors.CameraCfg`. The ``renderer_cfg`` field selects the rendering backend. - - -CameraCfg with renderer_cfg -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The renderer is specified via ``renderer_cfg`` on :class:`~sensors.CameraCfg`. The camera and -renderer configurations are fully decoupled: you can swap renderers without changing any other camera -parameters. - -**Default (RTX, requires Isaac Sim):** - -.. code-block:: python - - from isaaclab.sensors import CameraCfg - import isaaclab.sim as sim_utils - # IsaacRtxRendererCfg is the default, no explicit import needed - - tiled_camera: CameraCfg = CameraCfg( - prim_path="/World/envs/env_.*/Camera", - offset=CameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"), - data_types=["rgb"], - spawn=sim_utils.PinholeCameraCfg( - focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0) - ), - width=80, - height=80, - # renderer_cfg defaults to IsaacRtxRendererCfg() - ) - -**Newton Warp renderer (kit-less, no Isaac Sim required):** - -.. code-block:: python - - from isaaclab.sensors import CameraCfg - from isaaclab_newton.renderers import NewtonWarpRendererCfg - import isaaclab.sim as sim_utils - - tiled_camera: CameraCfg = CameraCfg( - prim_path="/World/envs/env_.*/Camera", - offset=CameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"), - data_types=["rgb", "depth"], # see the support matrix below for all Newton-supported types - spawn=sim_utils.PinholeCameraCfg( - focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0) - ), - width=80, - height=80, - renderer_cfg=NewtonWarpRendererCfg(), - ) - -**Multi-backend preset (switches renderer alongside physics backend):** - -For environments that need to support both backends, use -:class:`~isaaclab_tasks.utils.presets.MultiBackendRendererCfg` together with the -:ref:`PresetCfg pattern `: - -.. code-block:: python - - from isaaclab.sensors import CameraCfg - from isaaclab_tasks.utils.presets import MultiBackendRendererCfg - import isaaclab.sim as sim_utils - - tiled_camera: CameraCfg = CameraCfg( - prim_path="/World/envs/env_.*/Camera", - offset=CameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"), - data_types=["rgb"], - spawn=sim_utils.PinholeCameraCfg( - focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0) - ), - width=80, - height=80, - renderer_cfg=MultiBackendRendererCfg(), # selects RTX or Newton Warp via presets= CLI arg - ) - -The active preset is selected at launch via ``physics=``, ``renderer=``, or ``presets=`` CLI arguments: - -.. code-block:: bash - - # Use Newton Warp renderer - python train.py task=Isaac-Cartpole-Camera-Direct renderer=newton_renderer - - # Use OVRTX renderer - python train.py task=Isaac-Cartpole-Camera-Direct renderer=ovrtx - - # Use default (Isaac RTX) - python train.py task=Isaac-Cartpole-Camera-Direct - - -Accessing camera data -~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: python - - tiled_camera = Camera(cfg.tiled_camera) - data = tiled_camera.data.output["rgb"] # shape: (num_cameras, H, W, 3), torch.uint8 - -The returned data has shape ``(num_cameras, height, width, num_channels)``, ready to use directly -as an observation in RL training. - -Camera rendering is enabled automatically when launching an RTX camera task: - -.. tab-set:: - - .. tab-item:: uv (Recommended) - - .. code-block:: shell - - uv run --extra rl-games isaaclab train --rl_library rl_games \ - --task=Isaac-Cartpole-Camera-Direct - - - .. tab-item:: isaaclab.sh / isaaclab.bat - - .. code-block:: shell - - ./isaaclab.sh train --rl_library rl_games \ - --task=Isaac-Cartpole-Camera-Direct - - -Annotators -~~~~~~~~~~ - -:class:`~sensors.Camera` exposes the following annotator data types. Not every -backend produces every annotator — see the support matrix below. - -* ``"rgb"``: A 3-channel rendered color image. -* ``"rgba"``: A 4-channel rendered color image with alpha channel. -* ``"rgb_hdr"``: A 3-channel scene-linear HDR color image. -* ``"distance_to_camera"``: Distance to the camera optical center per pixel. -* ``"distance_to_image_plane"``: Distance along the camera's Z-axis per pixel. -* ``"depth"``: Alias for ``"distance_to_image_plane"``. -* ``"normals"``: Local surface normal vectors at each pixel. -* ``"motion_vectors"``: Per-pixel motion vectors in image space. -* ``"semantic_segmentation"``: Semantic segmentation labels. -* ``"instance_segmentation"``: Per-instance segmentation labels for semantically-tagged prims. -* ``"instance_id_segmentation_fast"``: Instance ID segmentation data. - -.. _camera-supported-annotators: - -Supported annotators by renderer backend -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following matrix shows which annotators each renderer backend can produce. -``Isaac RTX`` is :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`, ``OVRTX`` -is :class:`~isaaclab_ov.renderers.OVRTXRendererCfg`, and ``Newton Warp`` is -:class:`~isaaclab_newton.renderers.NewtonWarpRendererCfg`. - -.. list-table:: - :header-rows: 1 - :widths: 40 20 20 20 - - * - Data type - - Isaac RTX - - OVRTX - - Newton Warp - * - ``rgb`` - - ✅ - - ✅ - - ✅ - * - ``rgba`` - - ✅ - - ✅ - - ✅ - * - ``rgb_hdr`` - - ✅ - - ✅ - - ✅ - * - ``distance_to_camera`` - - ✅ - - ✅ - - ✅ - * - ``distance_to_image_plane`` - - ✅ - - ✅ - - ✅ - * - ``depth`` - - ✅ - - ✅ - - ✅ - * - ``normals`` - - ✅ - - ✅ - - ✅ - * - ``motion_vectors`` - - ✅ - - ✅ - - ❌ - * - ``semantic_segmentation`` - - ✅ - - ✅ - - ✅ - * - ``instance_segmentation`` - - ✅ - - ✅ - - ✅ - * - ``instance_id_segmentation_fast`` - - ✅ - - ❌ - - ❌ - -RGB and RGBA -~~~~~~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_rgb.jpg - :align: center - :figwidth: 100% - :alt: A scene captured in RGB - -``rgb`` returns a 3-channel RGB image of type ``torch.uint8``, shape ``(B, H, W, 3)``. - -``rgba`` returns a 4-channel RGBA image of type ``torch.uint8``, shape ``(B, H, W, 4)``. - -To convert to ``torch.float32``, divide by 255.0. - -``rgb_hdr`` returns a 3-channel scene-linear HDR image of type ``torch.float32``, shape ``(B, H, W, 3)``. - -Post-render Camera ISP -~~~~~~~~~~~~~~~~~~~~~~ - -A camera Image Signal Processing (ISP) pipeline models the chain that maps -the scene-linear radiance captured by a sensor to the LDR pixel values a -downstream consumer sees. -The camera ISP pipeline is usually part of the renderer. -In Isaac Lab we expose a post-render camera ISP pipeline which is applied on top of the renderer's HDR scene-linear AOV. -This makes it possible to implement additional post-render processing not currently supported by the renderer backends. -The pass is configured via :attr:`~sensors.CameraCfg.isp_cfg` -on every camera and runs once per render tick. - -PPISP -^^^^^ - -The shipped ISP implementation is **PPISP** (Physically Plausible Image -Signal Processing), an NVIDIA Spatial Intelligence Lab pipeline designed -to bring synthetic imagery — most notably 3D Gaussian splat reconstructions -— closer to real-camera output without re-training the upstream model. See -the project page: https://research.nvidia.com/labs/sil/projects/ppisp . - -PPISP is typically authored alongside a `ParticleField3DGaussianSplat -`__ -USD asset: its camera prim authors ``ppisp:*`` attributes calibrated against -the real capture rig that produced the splats. Configuring the camera with -the matching PPISP coefficients makes the rendered tile match the calibration -target. - -With static coefficients, the image pass applies, in order: responsivity → -exposure → vignetting → color homography → camera response function → uint8 -clamp. It runs as a single Warp image kernel. - -When the camera also authors ``ppisp:controllerWeights``, Isaac Lab runs the -exported PPISP controller before the image pass. The controller reads the -current HDR image, prior exposure, and controller responsivity, then predicts -``exposureOffset`` and the four color-latent pairs for each camera view. The -image pass then uses those predicted exposure and color-latent values while -static PPISP inputs still provide responsivity, vignetting, and CRF. - -Configuration -^^^^^^^^^^^^^ - -:attr:`~sensors.CameraCfg.isp_cfg` accepts three forms: - -* ``None`` (default) — ISP disabled. -* :class:`~isaaclab_ppisp.PpispCfg` — explicit PPISP coefficients - (:attr:`~isaaclab_ppisp.PpispCfg.inputs`) and, optionally, controller - weights. Use :attr:`~isaaclab_ppisp.PpispCfg.camera_prim_path` to import - static coefficients and camera-authored controller weights from a USD camera - already on the stage. -* :class:`~sensors.CameraISPMode` — auto-discover ISP camera attributes on the - stage (see below). - -The cfg applies once per Camera sensor batch. Static PPISP inputs are scalar -coefficients shared by every cloned view in a tiled batch. When controller -weights are present, the controller predicts per-view exposure and color-latent -values from each HDR image, while responsivity, vignetting, and CRF remain -shared by the batch. - -.. code-block:: python - - from isaaclab.sensors.camera import CameraCfg, CameraISPMode - from isaaclab_ppisp import PpispCfg - - # default — ISP disabled - cfg = CameraCfg(...) - - # explicit coefficients - cfg = CameraCfg(..., isp_cfg=PpispCfg(inputs={"exposureOffset": 1.5})) - - # import coefficients from a USD camera path - cfg = CameraCfg(..., isp_cfg=PpispCfg(camera_prim_path="/World/Camera_ppisp")) - - # auto-discover from the stage - cfg = CameraCfg(..., isp_cfg=CameraISPMode.AUTO_ANY) - -Auto-discovery -^^^^^^^^^^^^^^ - -Auto-discovery is opt-in via :class:`~sensors.CameraISPMode`. Discovery runs -once at camera construction using the first matched camera prim in the Camera -sensor batch: - -1. Check the first matched camera prim for ``ppisp:*`` attributes. -2. ``AUTO_ANY`` only, or when no camera path is available: fall back to the - first camera anywhere on the stage with ``ppisp:*`` attributes. -3. Otherwise the ISP stays disabled for the whole Camera sensor batch. - -In practice this means: if the stage carries a ``ParticleField3DGaussianSplat`` -together with a camera that authors ``ppisp:*`` attributes, the Camera sensor -picks up the matching ISP automatically, including controller weights when -authored, and no Python-side coefficient authoring is required. - -``AUTO_CAMERA`` runs only the camera-local discovery steps — useful when the -stage carries multiple PPISP cameras and you want the Camera sensor batch to use -the attributes authored on its first matched camera prim. - -Renderer support -^^^^^^^^^^^^^^^^ - -All three shipped backends advertise the HDR AOV -(:attr:`~renderers.RenderBufferKind.RGB_HDR`) and compose the ISP pipeline -internally: the Isaac RTX renderer sources HDR from the Replicator -``HdrColor`` annotator, the OVRTX renderer from its HDR render var, the -Newton Warp renderer from its native scene-linear color buffer. Each -backend allocates its own HDR scratch buffer when the user did not request -``"rgb_hdr"`` in :attr:`~sensors.CameraCfg.data_types`, and dispatches the -PPISP pipeline into ``rgb`` / ``rgba`` after every render tick. For controller -configs, this is the controller pass followed by the PPISP image pass. - -Usage example -^^^^^^^^^^^^^ - -For a runnable usage example, see ``scripts/demos/sensors/ppisp_camera.py``. -It loads a PPISP-authored USD or USDZ Gaussian scene, creates baseline and -PPISP camera sensors for the selected camera, and saves baseline, PPISP, and -absolute-difference images. - -.. tab-set:: - - .. tab-item:: uv (Recommended) - - .. code-block:: bash - - uv run --extra isaacsim python scripts/demos/sensors/ppisp_camera.py \ - --renderer newton_renderer --max_steps 60 - - .. tab-item:: isaaclab.sh / isaaclab.bat - - .. code-block:: bash - - ./isaaclab.sh -p scripts/demos/sensors/ppisp_camera.py \ - --renderer newton_renderer --max_steps 60 - -Use ``--renderer isaac_rtx`` to run the same workflow with Isaac RTX. Pass -``--input_scene`` for a custom scene and ``--camera_prim_path`` if the stage -contains multiple cameras with PPISP attributes. If a config or command selects -a visualizer, force-disable all visualizers with ``--visualizer none`` or -``--viz none``. Images are written to -``scripts/demos/sensors/output/ppisp_camera`` unless ``--output_dir`` is set. - -Known limitations -^^^^^^^^^^^^^^^^^ - -* The ISP writes back into the ``rgb`` / ``rgba`` buffers. If neither is - requested, configuring ``isp_cfg`` raises at camera init. -* Static PPISP inputs and controller weights are fixed for the lifetime of the - camera. Animated USD camera attributes are collapsed to their first authored - time sample. -* Static coefficients are global per Camera sensor batch — no per-pixel or - per-region authoring beyond the radial vignetting term. -* PPISP is the only ISP implementation today. Other ISP families would - need a new config type and discoverer entry. -* On the Isaac RTX and OVRTX backends, enabling ``isp_cfg`` forces RTX-side - tonemapping off (``/rtx/rtpt/gaussian/skipTonemapping/enabled=False``) - and authors a neutral ``OmniRtxCameraExposureAPI_1`` schema on each - camera prim so the post-render ISP is the only path that processes - color. Mixing this with RTX-side exposure authoring is not supported. -* Auto-discovery resolves at camera construction; later authoring of - ``ppisp:*`` camera attributes on the stage is not picked up. - -Depth and Distances -~~~~~~~~~~~~~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_depth.jpg - :align: center - :figwidth: 100% - :alt: A scene captured as depth - -``distance_to_camera`` returns a single-channel depth image with distance to the camera optical -center, shape ``(B, H, W, 1)``, type ``torch.float32``. - -``distance_to_image_plane`` returns distances of 3D points from the camera plane along the Z-axis, -shape ``(B, H, W, 1)``, type ``torch.float32``. - -``depth`` is an alias for ``distance_to_image_plane``. - -Normals -~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_normals.jpg - :align: center - :figwidth: 100% - :alt: A scene captured with surface normals - -``normals`` returns local surface normal vectors at each pixel, shape ``(B, H, W, 3)`` containing -``(x, y, z)``, type ``torch.float32``. - -Motion Vectors -~~~~~~~~~~~~~~ - -``motion_vectors`` returns per-pixel motion vectors in image space between frames. -Shape ``(B, H, W, 2)``: ``x`` is horizontal motion (positive = left), ``y`` is vertical motion -(positive = up). Type ``torch.float32``. - -Semantic Segmentation -~~~~~~~~~~~~~~~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_semantic.jpg - :align: center - :figwidth: 100% - :alt: A scene with semantic segmentation - -``semantic_segmentation`` outputs per-pixel semantic labels for entities with semantic annotations. -An ``info`` dictionary is available via ``tiled_camera.data.info['semantic_segmentation']``. - -- If ``colorize_semantic_segmentation=True``: 4-channel RGBA image, shape ``(B, H, W, 4)``, - type ``torch.uint8``. The ``idToLabels`` dict maps color to semantic label. -- If ``colorize_semantic_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``, - containing semantic IDs. The ``idToLabels`` dict maps ID to label. - -Instance ID Segmentation -~~~~~~~~~~~~~~~~~~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_instanceID.jpg - :align: center - :figwidth: 100% - :alt: A scene with instance ID segmentation - -``instance_id_segmentation_fast`` outputs per-pixel instance IDs, unique per USD prim path. -An ``info`` dictionary is available via ``tiled_camera.data.info['instance_id_segmentation_fast']``. - -- If ``colorize_instance_id_segmentation=True``: shape ``(B, H, W, 4)``, type ``torch.uint8``. - The ``idToLabels`` dict maps color to USD prim path. -- If ``colorize_instance_id_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``. - The ``idToLabels`` dict maps instance ID to USD prim path. - -Instance Segmentation -~~~~~~~~~~~~~~~~~~~~~ - -.. figure:: ../../../_static/overview/sensors/camera_instance.jpg - :align: center - :figwidth: 100% - :alt: A scene with instance segmentation - -``instance_segmentation`` outputs instance segmentation, traversing down the prim hierarchy -to the lowest level with semantic labels (unlike ``instance_id_segmentation_fast``, which always -goes to the leaf prim). -An ``info`` dictionary is available via ``tiled_camera.data.info['instance_segmentation']``. - -- If ``colorize_instance_segmentation=True``: shape ``(B, H, W, 4)``, type ``torch.uint8``. -- If ``colorize_instance_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``. - -Pixels belonging to prims with no assigned semantic label are rendered black -(RGBA ``(0, 0, 0, 255)``) when ``colorize_instance_segmentation=True``. When -``colorize_instance_segmentation=False``, those pixels instead carry the raw -UNLABELLED instance ID (``1``) rather than a color value. - -The ``idToLabels`` dict maps color to USD prim path. The ``idToSemantics`` dict maps color to -semantic label. - -Background Color -~~~~~~~~~~~~~~~~ - -By default, pixels that do not intersect any geometry use the renderer's built-in background: -a dome-light sky on RTX backends, or a neutral gray on Newton Warp. You can override this with -a solid color by setting :attr:`~sensors.CameraCfg.background_color` on :class:`~sensors.CameraCfg`: - -.. code-block:: python - - from isaaclab.sensors import CameraCfg - import isaaclab.sim as sim_utils - - tiled_camera = CameraCfg( - prim_path="/World/envs/env_.*/Camera", - data_types=["rgb"], - spawn=sim_utils.PinholeCameraCfg( - focal_length=24.0, focus_distance=400.0, - horizontal_aperture=20.955, clipping_range=(0.1, 20.0), - ), - width=80, - height=80, - background_color=(0.0, 0.0, 0.0), # black background - ) - -The color is a 3-tuple of normalized RGB floats in ``[0, 1]``. Setting it to ``None`` (the -default) leaves the renderer's built-in background in place. - -Each camera is configured independently: cameras with ``background_color=None`` and cameras -with a color set can coexist in the same scene. On the RTX backends (Isaac RTX and OVRTX) this -is achieved by writing ``omni:rtx:background:source:type`` / ``omni:rtx:background:source:color`` -USD attributes directly on the render product, so no process-wide carb settings are modified. - -.. list-table:: - :header-rows: 1 - :widths: 30 70 - - * - Backend - - Behavior when ``background_color`` is set - * - ``IsaacRtxRendererCfg`` - - Per-render-product USD attributes on the Replicator render product. - * - ``OVRTXRendererCfg`` - - Per-render-product USD attributes authored in the render scope. - * - ``NewtonWarpRendererCfg`` - - Packed ARGB clear color used to fill the framebuffer before rasterization. diff --git a/docs/source/overview/core-concepts/sensors/contact_sensor.rst b/docs/source/overview/core-concepts/sensors/contact_sensor.rst deleted file mode 100644 index 85d60397d8f2..000000000000 --- a/docs/source/overview/core-concepts/sensors/contact_sensor.rst +++ /dev/null @@ -1,114 +0,0 @@ -.. _overview_sensors_contact: - -.. currentmodule:: isaaclab - -Contact Sensor -============== - -.. figure:: ../../../_static/overview/sensors/contact_diagram.jpg - :align: center - :figwidth: 100% - :alt: A contact sensor with filtering - -The contact sensor is designed to return contact forces acting on a given rigid body. The sensor is written to behave as a physical object, and so the "scope" of the contact sensor is limited to the body (or bodies) that defines it. There are multiple ways to define this scope, depending on your need to filter the forces coming from the contact. - -The aggregate normal force is reported as ``net_normal_forces_w``. When supported by the backend and enabled with ``track_friction_forces``, the aggregate friction force is reported as ``net_friction_forces_w``. The total contact force is therefore - -.. math:: - - \boldsymbol{f}_{total} = \boldsymbol{f}_{normal} + \boldsymbol{f}_{friction}. - -On Newton, ``net_forces_w`` is this total. PhysX and OVPhysX cannot compute a total contact force, so ``net_forces_w`` returns ``net_normal_forces_w`` and warns; this is a known limitation planned to be fixed in a later release. Use the explicit normal / friction properties when the split matters. History buffers are available for both normal and friction quantities (``net_normal_forces_w_history``, ``net_friction_forces_w_history``, ``normal_force_matrix_w_history``, and ``friction_force_matrix_w_history``). - -Your application may only care about contact forces due to specific objects. Retrieving contact forces from specific objects requires filtering. The per-filter normal and friction values are exposed as ``normal_force_matrix_w`` and ``friction_force_matrix_w``. Summing a force matrix over its filter dimension only reconstructs the corresponding net force when the filters cover every contacting object. - -PhysX does not expose an unfiltered aggregate friction force through its tensor API. Accessing ``net_friction_forces_w`` raises ``NotImplementedError``. ``friction_forces_w`` is that aggregate quantity; on PhysX it returns ``friction_force_matrix_w`` and warns. Newton exposes both aggregate and filtered friction forces. - -Consider a simple environment with an Anymal Quadruped and a block - -.. literalinclude:: ../../../../../scripts/demos/sensors/contact_sensor.py - :language: python - :lines: 40-90 - -We define the sensors on the feet of the robot in two different ways. The front feet are independent sensors (one sensor body per foot) and the "Cube" is placed under the left foot. The hind feet are defined as a single sensor with multiple bodies. - -We can then run the scene and print the data from the sensors - -.. code-block:: python - - def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): - . - . - . - # Simulate physics - while simulation_app.is_running(): - . - . - . - # print information from the sensors - print("-------------------------------") - print(scene["contact_forces_LF"]) - print("Received force matrix of: ", scene["contact_forces_LF"].data.normal_force_matrix_w) - print("Received contact force of: ", scene["contact_forces_LF"].data.net_normal_forces_w) - print("-------------------------------") - print(scene["contact_forces_RF"]) - print("Received force matrix of: ", scene["contact_forces_RF"].data.normal_force_matrix_w) - print("Received contact force of: ", scene["contact_forces_RF"].data.net_normal_forces_w) - print("-------------------------------") - print(scene["contact_forces_H"]) - print("Received force matrix of: ", scene["contact_forces_H"].data.normal_force_matrix_w) - print("Received contact force of: ", scene["contact_forces_H"].data.net_normal_forces_w) - -Here, we print both the net contact force and the filtered force matrix for each contact sensor defined in the scene. The front left and front right feet report the following - -.. code-block:: bash - - ------------------------------- - Contact sensor @ '/World/envs/env_.*/Robot/LF_FOOT': - view type : - update period (s) : 0.0 - number of bodies : 1 - body names : ['LF_FOOT'] - - Received force matrix of: tensor([[[[-1.3923e-05, 1.5727e-04, 1.1032e+02]]]], device='cuda:0') - Received contact force of: tensor([[[-1.3923e-05, 1.5727e-04, 1.1032e+02]]], device='cuda:0') - ------------------------------- - Contact sensor @ '/World/envs/env_.*/Robot/RF_FOOT': - view type : - update period (s) : 0.0 - number of bodies : 1 - body names : ['RF_FOOT'] - - Received force matrix of: tensor([[[[0., 0., 0.]]]], device='cuda:0') - Received contact force of: tensor([[[1.3529e-05, 0.0000e+00, 1.0069e+02]]], device='cuda:0') - - -.. figure:: ../../../_static/overview/sensors/contact_visualization.jpg - :align: center - :figwidth: 100% - :alt: The contact sensor visualization - - -Notice that even with filtering, both sensors report the net contact force acting on the foot. However, the "force matrix" on the right foot is zero because that foot isn't in contact with the filtered body, ``/World/envs/env_.*/Cube``. Now, checkout the data coming from the hind feet! - -.. code-block:: bash - - ------------------------------- - Contact sensor @ '/World/envs/env_.*/Robot/.*H_FOOT': - view type : - update period (s) : 0.0 - number of bodies : 2 - body names : ['LH_FOOT', 'RH_FOOT'] - - Received force matrix of: None - Received contact force of: tensor([[[9.7227e-06, 0.0000e+00, 7.2364e+01], - [2.4322e-05, 0.0000e+00, 1.8102e+02]]], device='cuda:0') - -In this case, the contact sensor has two bodies: the left and right hind feet. When the force matrix is queried, the result is ``None`` because this is a many body sensor, and presently Isaac Lab only supports "many to one" contact force filtering. Unlike the single body contact sensor, the reported force tensor has multiple entries, with each "row" corresponding to the contact force on a single body of the sensor (matching the ordering at construction). - -.. dropdown:: Code for contact_sensor.py - :icon: code - - .. literalinclude:: ../../../../../scripts/demos/sensors/contact_sensor.py - :language: python - :linenos: diff --git a/docs/source/overview/core-concepts/sensors/frame_transformer.rst b/docs/source/overview/core-concepts/sensors/frame_transformer.rst deleted file mode 100644 index 7ebe0020cdb1..000000000000 --- a/docs/source/overview/core-concepts/sensors/frame_transformer.rst +++ /dev/null @@ -1,111 +0,0 @@ -.. _overview_sensors_frame_transformer: - -.. currentmodule:: isaaclab - -Frame Transformer -================= - -.. figure:: ../../../_static/overview/sensors/frame_transformer.jpg - :align: center - :figwidth: 100% - :alt: A diagram outlining the basic geometry of frame transformations - -.. - Do YOU want to know where things are relative to other things at a glance? Then the frame transformer is the sensor for you!* - -One of the most common operations that needs to be performed within a physics simulation is the frame transformation: rewriting a vector or quaternion in the basis of an arbitrary euclidean coordinate system. There are many ways to accomplish this within Isaac and USD, but these methods can be cumbersome to implement within Isaac Lab's GPU based simulation and cloned environments. To mitigate this problem, we have designed the Frame Transformer Sensor, that tracks and calculate the relative frame transformations for rigid bodies of interest to the scene. - -The sensory is minimally defined by a source frame and a list of target frames. These definitions take the form of a prim path (for the source) and list of regex capable prim paths the rigid bodies to be tracked (for the targets). - -.. literalinclude:: ../../../../../scripts/demos/sensors/frame_transformer_sensor.py - :language: python - :lines: 38-86 - -We can now run the scene and query the sensor for data - -.. code-block:: python - - def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): - . - . - . - # Simulate physics - while simulation_app.is_running(): - . - . - . - - # print information from the sensors - print("-------------------------------") - print(scene["specific_transforms"]) - print("relative transforms:", scene["specific_transforms"].data.target_pos_source) - print("relative orientations:", scene["specific_transforms"].data.target_quat_source) - print("-------------------------------") - print(scene["cube_transform"]) - print("relative transform:", scene["cube_transform"].data.target_pos_source) - print("-------------------------------") - print(scene["robot_transforms"]) - print("relative transforms:", scene["robot_transforms"].data.target_pos_source) - -Let's take a look at the result for tracking specific objects. First, we can take a look at the data coming from the -sensors on the feet - -.. code-block:: bash - - ------------------------------- - FrameTransformer @ '/World/envs/env_.*/Robot/base': - tracked body frames: ['base', 'LF_FOOT', 'RF_FOOT'] - number of envs: 1 - source body frame: base - target frames (count: ['LF_FOOT', 'RF_FOOT']): 2 - - relative transforms: tensor([[[ 0.4658, 0.3085, -0.4840], - [ 0.4487, -0.2959, -0.4828]]], device='cuda:0') - relative orientations: tensor([[[ 0.9623, 0.0072, -0.2717, -0.0020], - [ 0.9639, 0.0052, -0.2663, -0.0014]]], device='cuda:0') - -.. figure:: ../../../_static/overview/sensors/frame_transformer_visualizer.jpg - :align: center - :figwidth: 100% - :alt: The frame transformer visualizer - -By activating the visualizer, we can see that the frames of the feet are rotated "upward" slightly. -We can also see the explicit relative positions and rotations by querying the sensor for data, which -returns these values as a list with the same order as the tracked frames. This becomes even more -apparent if we examine the transforms specified by regex. - -.. code-block:: bash - - ------------------------------- - FrameTransformer @ '/World/envs/env_.*/Robot/base': - tracked body frames: ['base', 'LF_FOOT', 'LF_HIP', 'LF_SHANK', 'LF_THIGH', 'LH_FOOT', 'LH_HIP', 'LH_SHANK', 'LH_THIGH', 'RF_FOOT', 'RF_HIP', 'RF_SHANK', 'RF_THIGH', 'RH_FOOT', 'RH_HIP', 'RH_SHANK', 'RH_THIGH', 'base'] - number of envs: 1 - source body frame: base - target frames (count: ['LF_FOOT', 'LF_HIP', 'LF_SHANK', 'LF_THIGH', 'LH_FOOT', 'LH_HIP', 'LH_SHANK', 'LH_THIGH', 'RF_FOOT', 'RF_HIP', 'RF_SHANK', 'RF_THIGH', 'RH_FOOT', 'RH_HIP', 'RH_SHANK', 'RH_THIGH', 'base']): 17 - - relative transforms: tensor([[[ 4.6581e-01, 3.0846e-01, -4.8398e-01], - [ 2.9990e-01, 1.0400e-01, -1.7062e-09], - [ 2.1409e-01, 2.9177e-01, -2.4214e-01], - [ 3.5980e-01, 1.8780e-01, 1.2608e-03], - [-4.8813e-01, 3.0973e-01, -4.5927e-01], - [-2.9990e-01, 1.0400e-01, 2.7044e-09], - [-2.1495e-01, 2.9264e-01, -2.4198e-01], - [-3.5980e-01, 1.8780e-01, 1.5582e-03], - [ 4.4871e-01, -2.9593e-01, -4.8277e-01], - [ 2.9990e-01, -1.0400e-01, -2.7057e-09], - [ 1.9971e-01, -2.8554e-01, -2.3778e-01], - [ 3.5980e-01, -1.8781e-01, -9.1049e-04], - [-5.0090e-01, -2.9095e-01, -4.5746e-01], - [-2.9990e-01, -1.0400e-01, 6.3592e-09], - [-2.1860e-01, -2.8251e-01, -2.5163e-01], - [-3.5980e-01, -1.8779e-01, -1.8792e-03], - [ 0.0000e+00, 0.0000e+00, 0.0000e+00]]], device='cuda:0') - -Here, the sensor is tracking all rigid body children of ``Robot/base``, but this expression is **inclusive**, meaning that the source body itself is also a target. This can be seen both by examining the source and target list, where ``base`` appears twice, and also in the returned data, where the sensor returns the relative transform to itself, (0, 0, 0). - -.. dropdown:: Code for frame_transformer_sensor.py - :icon: code - - .. literalinclude:: ../../../../../scripts/demos/sensors/frame_transformer_sensor.py - :language: python - :linenos: diff --git a/docs/source/overview/core-concepts/sensors/imu.rst b/docs/source/overview/core-concepts/sensors/imu.rst deleted file mode 100644 index 82e827ea2251..000000000000 --- a/docs/source/overview/core-concepts/sensors/imu.rst +++ /dev/null @@ -1,89 +0,0 @@ -.. _overview_sensors_imu: - -.. currentmodule:: isaaclab - -Inertial Measurement Unit (IMU) -=================================== - -.. figure:: ../../../_static/overview/sensors/imu_diagram.jpg - :align: center - :figwidth: 100% - :alt: A diagram outlining the basic force relationships for the IMU sensor - -Inertial Measurement Units (IMUs) are a type of sensor for measuring the acceleration of an object. These sensors are traditionally designed report linear accelerations and angular velocities, and function on similar principles to that of a digital scale: They report accelerations derived from **net force acting on the sensor**. - -A naive implementation of an IMU would report a negative acceleration due to gravity while the sensor is at rest in some local gravitational field. This is not generally needed for most practical applications, and so most real IMU sensors often include a **gravity bias** and assume that the device is operating on the surface of the Earth. The IMU we provide in Isaac Lab includes a similar bias term, which defaults to +g. This means that if you add an IMU to your simulation, and do not change this bias term, you will detect an acceleration of :math:`+ 9.81 m/s^{2}` anti-parallel to gravity acceleration. - -Consider a simple environment with an Anymal Quadruped equipped with an IMU on each of its two front feet. - -.. literalinclude:: ../../../../../scripts/demos/sensors/imu_sensor.py - :language: python - :lines: 39-63 - -Here we have explicitly removed the bias from one of the sensors, and we can see how this affects the reported values by visualizing the sensor when we run the sample script - -.. figure:: ../../../_static/overview/sensors/imu_visualizer.jpg - :align: center - :figwidth: 100% - :alt: IMU visualized - -Notice that the right front foot explicitly has a bias of (0,0,0). In the visualization, you should see that the arrow indicating the acceleration from the right IMU rapidly changes over time, while the arrow visualizing the left IMU points constantly along the vertical axis. - -Retrieving values from the sensor is done in the usual way - -.. code-block:: python - - def run_simulator(sim: sim_utils.SimulationContext, scene: InteractiveScene): - . - . - . - # Simulate physics - while simulation_app.is_running(): - . - . - . - # print information from the sensors - print("-------------------------------") - print(scene["imu_LF"]) - print("Received linear velocity: ", scene["imu_LF"].data.lin_vel_b) - print("Received angular velocity: ", scene["imu_LF"].data.ang_vel_b) - print("Received linear acceleration: ", scene["imu_LF"].data.lin_acc_b) - print("Received angular acceleration: ", scene["imu_LF"].data.ang_acc_b) - print("-------------------------------") - print(scene["imu_RF"]) - print("Received linear velocity: ", scene["imu_RF"].data.lin_vel_b) - print("Received angular velocity: ", scene["imu_RF"].data.ang_vel_b) - print("Received linear acceleration: ", scene["imu_RF"].data.lin_acc_b) - print("Received angular acceleration: ", scene["imu_RF"].data.ang_acc_b) - -The oscillations in the values reported by the sensor are a direct result of how the sensor calculates the acceleration, which is through a finite difference approximation between adjacent ground truth velocity values as reported by the sim. We can see this in the reported result (pay attention to the **linear acceleration**) because the acceleration from the right foot is small, but explicitly zero. - -.. code-block:: bash - - Imu sensor @ '/World/envs/env_.*/Robot/LF_FOOT': - view type : - update period (s) : 0.0 - number of sensors : 1 - - Received linear velocity: tensor([[ 0.0203, -0.0054, 0.0380]], device='cuda:0') - Received angular velocity: tensor([[-0.0104, -0.1189, 0.0080]], device='cuda:0') - Received linear acceleration: tensor([[ 4.8344, -0.0205, 8.5305]], device='cuda:0') - Received angular acceleration: tensor([[-0.0389, -0.0262, -0.0045]], device='cuda:0') - ------------------------------- - Imu sensor @ '/World/envs/env_.*/Robot/RF_FOOT': - view type : - update period (s) : 0.0 - number of sensors : 1 - - Received linear velocity: tensor([[0.0244, 0.0077, 0.0431]], device='cuda:0') - Received angular velocity: tensor([[ 0.0122, -0.1360, -0.0042]], device='cuda:0') - Received linear acceleration: tensor([[-0.0018, 0.0010, -0.0032]], device='cuda:0') - Received angular acceleration: tensor([[-0.0373, -0.0050, -0.0053]], device='cuda:0') - ------------------------------- - -.. dropdown:: Code for imu_sensor.py - :icon: code - - .. literalinclude:: ../../../../../scripts/demos/sensors/imu_sensor.py - :language: python - :linenos: diff --git a/docs/source/overview/core-concepts/sensors/index.rst b/docs/source/overview/core-concepts/sensors/index.rst deleted file mode 100644 index 436d11a81e47..000000000000 --- a/docs/source/overview/core-concepts/sensors/index.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _overview_sensors: - -Sensors -========= - -.. seealso:: - - This section is the source of truth for the ``isaaclab-using-sensors-actuators`` agent skill - (`skills/user/use-sensors-actuators/ <../../../../../skills/user/use-sensors-actuators/SKILL.md>`__). - When you change these pages, update the skill so agent guidance stays in sync. See - :doc:`/source/overview/developer-guide/agent_skills`. - -In this section, we will overview the various sensor APIs provided by Isaac Lab. - -Every sensor in Isaac Lab inherits from the ``SensorBase`` abstract class that provides the core functionality inherent to all sensors, which is to provide access to "measurements" of the scene. These measurements can take many forms such as ray-casting results, camera rendered images, or even simply ground truth data queried directly from the simulation (such as poses). Whatever the data may be, we can think of the sensor as having a buffer that is periodically updated with measurements by querying the scene. This ``update_period`` is defined in "simulated" seconds, meaning that even if the flow of time in the simulation is dilated relative to the real world, the sensor will update at the appropriate rate. The ``SensorBase`` is also designed with vectorizability in mind, holding the buffers for all copies of the sensor across cloned environments. - -Updating the buffers is done by overriding the ``_update_buffers_impl`` abstract method of the ``SensorBase`` class. On every time-step of the simulation, ``dt``, all sensors are queried for an update. During this query, the total time since the last update is incremented by ``dt`` for every buffer managed by that particular sensor. If the total time is greater than or equal to the ``update_period`` for a buffer, then that buffer is flagged to be updated on the next query. - -The following pages describe the available sensors in more detail: - -.. toctree:: - :maxdepth: 1 - - camera - contact_sensor - frame_transformer - imu - pva - joint_wrench_sensor - ray_caster diff --git a/docs/source/overview/core-concepts/sensors/joint_wrench_sensor.rst b/docs/source/overview/core-concepts/sensors/joint_wrench_sensor.rst deleted file mode 100644 index 9c441b215da8..000000000000 --- a/docs/source/overview/core-concepts/sensors/joint_wrench_sensor.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. _overview_sensors_joint_wrench: - -.. currentmodule:: isaaclab - -Joint Wrench Sensor -=================== - -The joint wrench sensor reports incoming joint reaction wrenches for selected -articulation bodies. It exposes force [N] and torque [N·m] buffers separately, -with entries ordered by the sensor's :attr:`~isaaclab.sensors.JointWrenchSensor.body_names`. -The default convention is ``incoming_joint_frame``, which expresses each wrench -in the child-side joint frame at the child-side joint anchor. - -The sensor is configured on an articulation prim and can then be used directly -or through manager terms such as :func:`~isaaclab.envs.mdp.body_incoming_wrench`. -For example, the Ant environment adds a joint wrench sensor to the scene: - -.. literalinclude:: ../../../../../source/isaaclab_tasks/isaaclab_tasks/core/locomotion/ant/ant_manager_env_cfg.py - :language: python - :lines: 91-95 - -The same environment uses :class:`~isaaclab.managers.SceneEntityCfg` to select -the reported foot bodies for an observation term: - -.. literalinclude:: ../../../../../source/isaaclab_tasks/isaaclab_tasks/core/locomotion/ant/ant_manager_env_cfg.py - :language: python - :lines: 133-142 - -Direct access to the sensor data follows the usual scene lookup pattern. - -.. code-block:: python - - joint_wrench = scene["joint_wrench"] - foot_ids, _ = joint_wrench.find_bodies([".*foot"]) - - force = joint_wrench.data.force.torch[:, foot_ids] - torque = joint_wrench.data.torque.torch[:, foot_ids] - wrench = torch.cat((force, torque), dim=-1) - -The resulting ``wrench`` tensor has shape ``(num_envs, num_selected_bodies, 6)`` -and stores the force components followed by the torque components for each -selected body. diff --git a/docs/source/overview/core-concepts/sensors/pva.rst b/docs/source/overview/core-concepts/sensors/pva.rst deleted file mode 100644 index 5061b38ea0f7..000000000000 --- a/docs/source/overview/core-concepts/sensors/pva.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. _overview_sensors_pva: - -.. currentmodule:: isaaclab - -Pose Velocity Acceleration (PVA) Sensor -======================================= - -The Pose Velocity Acceleration (PVA) sensor is a ground-truth sensor for reading -the kinematic state of a frame in the simulation. It reports the sensor pose in -the world frame, projected gravity, linear and angular velocities in the sensor -frame, and coordinate accelerations in the sensor frame. Unlike -:class:`~isaaclab.sensors.Imu`, the PVA sensor does not model proper -acceleration from an accelerometer. Use the IMU sensor when the observation -should include accelerometer-like gravity bias behavior. - -The sensor can be attached to a rigid body or to a child prim under a rigid-body -ancestor. If the configured prim is not itself rigid, Isaac Lab queries the -closest rigid ancestor and composes the fixed transform to the requested prim -with the configured sensor offset. - -Consider a simple environment with an Anymal Quadruped equipped with PVA sensors -on its front feet. - -.. literalinclude:: ../../../../../scripts/demos/sensors/pva_sensor.py - :language: python - :lines: 43-59 - -Retrieving values from the sensor follows the same pattern as the other Isaac -Lab sensors. The data fields are exposed as :class:`~isaaclab.utils.warp.ProxyArray` -buffers and can be converted to Torch tensors with the ``torch`` property. - -.. code-block:: python - - pva_data = scene["pva_LF"].data - print("Pose in world frame: ", pva_data.pose_w.torch) - print("Linear velocity in PVA frame: ", pva_data.lin_vel_b.torch) - print("Angular velocity in PVA frame: ", pva_data.ang_vel_b.torch) - print("Linear acceleration in PVA frame: ", pva_data.lin_acc_b.torch) - print("Angular acceleration in PVA frame: ", pva_data.ang_acc_b.torch) - print("Projected gravity in PVA frame: ", pva_data.projected_gravity_b.torch) - -The complete demo can be run with: - -.. tab-set:: - - .. tab-item:: uv (Recommended) - - .. code-block:: bash - - uv run --extra isaacsim python scripts/demos/sensors/pva_sensor.py - - .. tab-item:: isaaclab.sh / isaaclab.bat - - .. code-block:: bash - - ./isaaclab.sh -p scripts/demos/sensors/pva_sensor.py - -.. dropdown:: Code for pva_sensor.py - :icon: code - - .. literalinclude:: ../../../../../scripts/demos/sensors/pva_sensor.py - :language: python - :linenos: diff --git a/skills/user/use-sensors-actuators/SKILL.md b/skills/user/use-sensors-actuators/SKILL.md index f9db187123d9..3eb84ed050c7 100644 --- a/skills/user/use-sensors-actuators/SKILL.md +++ b/skills/user/use-sensors-actuators/SKILL.md @@ -49,7 +49,7 @@ uv run --no-project python tools/skills/cli.py check ## Maintenance Keep this skill synchronized with actuator docs under `docs/source/concepts/`, sensor docs under -`docs/source/overview/core-concepts/sensors/`, sensor tutorials under +`docs/source/concepts/sensors/`, sensor tutorials under `docs/source/tutorials/04_sensors/`, API docs under `docs/source/api/`, and maintained task examples under `source/isaaclab_tasks/isaaclab_tasks/`. If sensor support changes for PhysX, Newton, or renderers, update the docs or source examples first. @@ -59,8 +59,8 @@ renderers, update the docs or source examples first. - [Evaluations](evaluations.md) - [Examples](examples.md) - [Add sensors on robot tutorial](../../../docs/source/tutorials/04_sensors/add_sensors_on_robot.rst) -- [Contact sensor docs](../../../docs/source/overview/core-concepts/sensors/contact_sensor.rst) -- [Joint wrench sensor docs](../../../docs/source/overview/core-concepts/sensors/joint_wrench_sensor.rst) +- [Contact sensor docs](../../../docs/source/concepts/sensors/contact_sensor.rst) +- [Joint wrench sensor docs](../../../docs/source/concepts/sensors/joint_wrench_sensor.rst) - [Actuators docs](../../../docs/source/concepts/actuators.rst) - [Sensors API](../../../docs/source/api/lab/isaaclab.sensors.rst) - [Sensor patterns API](../../../docs/source/api/lab/isaaclab.sensors.patterns.rst) diff --git a/tools/docs/media/README.md b/tools/docs/media/README.md index 296593cebbb5..9abdfe4250a2 100644 --- a/tools/docs/media/README.md +++ b/tools/docs/media/README.md @@ -22,3 +22,24 @@ compatible completed run, set `RL_PROGRESS_CHECKPOINT_DIR` to a run directory co The generators require `uv`, `ffmpeg`, a CUDA-capable GPU, and the optional runtime packages installed by their `uv run --extra` commands. + +## Renderer gallery + +The renderer gallery compares Newton Warp, OVRTX, and Isaac RTX camera outputs. +It uses the published Nucleus stage by default when regenerating all RGB +animations and still output modes: + +```bash +OMNI_KIT_ACCEPT_EULA=Y tools/docs/media/generate_renderer_gallery.sh +``` + +Pass a local path or another Nucleus URI to override the default stage: + +```bash +OMNI_KIT_ACCEPT_EULA=Y tools/docs/media/generate_renderer_gallery.sh \ + /path/to/renderer-gallery-scene.usda +``` + +The generator launches the kit-less renderers separately from Isaac RTX because +their optional runtime packages cannot share one process. It writes the final +WebP and PNG assets directly to `docs/source/_static/overview/sensors`. diff --git a/tools/docs/media/capture_renderer_gallery.py b/tools/docs/media/capture_renderer_gallery.py new file mode 100644 index 000000000000..600a5c4a59a4 --- /dev/null +++ b/tools/docs/media/capture_renderer_gallery.py @@ -0,0 +1,435 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Capture the renderer documentation gallery from an editable USD scene.""" + +from __future__ import annotations + +import argparse +import contextlib +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class GalleryMode: + """One visually distinct camera output included in the gallery.""" + + output_name: str + label: str + animated: bool = False + + +_COMMON_MODES = ( + GalleryMode("rgb", "RGB", animated=True), + GalleryMode("albedo", "Albedo"), + GalleryMode("depth", "Depth"), + GalleryMode("normals", "Normals"), + GalleryMode("semantic_segmentation", "Semantic segmentation"), + GalleryMode("instance_segmentation", "Instance segmentation"), +) +_RTX_ONLY_MODES = ( + GalleryMode("motion_vectors", "Motion vectors"), + GalleryMode("simple_shading_constant_diffuse", "Constant diffuse"), + GalleryMode("simple_shading_diffuse_mdl", "Diffuse MDL"), + GalleryMode("simple_shading_full_mdl", "Full MDL"), +) +_RENDERER_SLUGS = {"newton": "newton", "ovrtx": "ovrtx", "isaac_rtx": "isaac-rtx"} +_SIMPLE_SHADING_MODES = tuple(mode.output_name for mode in _RTX_ONLY_MODES if mode.output_name.startswith("simple_")) +_OVRTX_AMBIENT_LIGHT_SETTING = "float omni:rtx:rt:ambientLight:intensity = 1.0" +_DEFAULT_GALLERY_SCENE = "omniverse://isaac-dev.ov.nvidia.com/Isaac/IsaacLab/Docs/Renderers/renderer_gallery_scene.usda" + + +def gallery_modes(renderer: str) -> tuple[GalleryMode, ...]: + """Return the visually distinct outputs documented for a renderer backend.""" + if renderer == "newton": + return _COMMON_MODES + if renderer in {"ovrtx", "isaac_rtx"}: + return (*_COMMON_MODES, *_RTX_ONLY_MODES) + raise ValueError(f"Unknown renderer: {renderer}") + + +def capture_data_types(renderer: str, capture_group: str) -> tuple[str, ...]: + """Return the camera outputs captured together for one renderer process.""" + modes = gallery_modes(renderer) + if capture_group == "standard": + return tuple(mode.output_name for mode in modes if not mode.output_name.startswith("simple_shading_")) + if any(mode.output_name == capture_group for mode in modes) and capture_group.startswith("simple_shading_"): + return (capture_group,) + raise ValueError(f"Capture group {capture_group!r} is not available for renderer {renderer!r}.") + + +def gallery_asset_name(renderer: str, output_name: str) -> str: + """Return the published documentation asset name for one renderer output.""" + try: + renderer_slug = _RENDERER_SLUGS[renderer] + except KeyError as exc: + raise ValueError(f"Unknown renderer: {renderer}") from exc + suffix = "webp" if output_name == "rgb" else "png" + output_suffix = "" if output_name == "rgb" else f"-{output_name.replace('_', '-')}" + return f"camera-renderer-{renderer_slug}{output_suffix}.{suffix}" + + +def snapshot_camera_tensor(data: Any) -> Any: + """Copy a renderer-owned tensor to stable CPU storage.""" + return data.detach().to(device="cpu", copy=True) + + +def depth_display_bounds(data: Any) -> tuple[float, float]: + """Return the minimum and maximum finite depths in a frame.""" + import torch + + finite_depth = data[torch.isfinite(data)] + if finite_depth.numel() == 0: + raise ValueError("Depth frame does not contain finite samples.") + return float(finite_depth.min()), float(finite_depth.max()) + + +def motion_vectors_to_image(data: Any) -> Any: + """Colorize motion vectors and overlay sparse image-space direction arrows.""" + import numpy as np + import torch + from PIL import Image, ImageDraw + + data = snapshot_camera_tensor(data) + uv = data[..., :2].float() + raw_magnitude = torch.linalg.vector_norm(uv, dim=-1) + max_magnitude = max(float(raw_magnitude.quantile(0.99)), 1.0e-6) + normalized_uv = (uv / max_magnitude).clamp(-1.0, 1.0) + normalized_magnitude = torch.linalg.vector_norm(normalized_uv, dim=-1).clamp(0.0, 1.0) + array = ( + torch.cat(((normalized_uv + 1.0) * 0.5, normalized_magnitude.unsqueeze(-1)), dim=-1) + .mul(255) + .to(torch.uint8) + .numpy() + ) + image = Image.fromarray(np.ascontiguousarray(array).copy(), mode="RGB") + draw = ImageDraw.Draw(image) + + uv_array = normalized_uv.numpy() + magnitude_array = normalized_magnitude.numpy() + height, width = magnitude_array.shape + grid_spacing = max(min(height, width) // 10, 16) + arrow_length = grid_spacing * 0.45 + head_length = max(grid_spacing * 0.18, 4.0) + + for cell_y in range(0, height, grid_spacing): + for cell_x in range(0, width, grid_spacing): + cell = magnitude_array[ + cell_y : min(cell_y + grid_spacing, height), + cell_x : min(cell_x + grid_spacing, width), + ] + if cell.size == 0 or float(cell.max()) < 0.08: + continue + local_y, local_x = np.unravel_index(int(cell.argmax()), cell.shape) + y = cell_y + int(local_y) + x = cell_x + int(local_x) + direction = uv_array[y, x].copy() + # Image-space v points up, while raster screen y points down. + direction[1] *= -1.0 + direction_length = float(np.linalg.norm(direction)) + if direction_length < 1.0e-6: + continue + + unit = direction / direction_length + end = np.array((x, y), dtype=np.float32) + unit * arrow_length * float(magnitude_array[y, x]) + start_xy = (float(x), float(y)) + end_xy = (float(end[0]), float(end[1])) + draw.line((start_xy, end_xy), fill=(0, 0, 0), width=4) + draw.line((start_xy, end_xy), fill=(255, 255, 255), width=2) + + perpendicular = np.array((-unit[1], unit[0]), dtype=np.float32) + head_base = end - unit * head_length + head_half_width = head_length * 0.55 + draw.polygon( + ( + end_xy, + tuple(head_base + perpendicular * head_half_width), + tuple(head_base - perpendicular * head_half_width), + ), + fill=(255, 255, 255), + ) + + return image + + +def thumbnail_frame_index(frame_count: int) -> int: + """Select the sixth captured frame so temporal outputs have useful motion history.""" + if frame_count < 6: + raise ValueError("At least six animation frames are required to capture thumbnails.") + return 5 + + +def renderer_requires_kit(renderer: str) -> bool: + """Return whether a renderer must run inside Isaac Sim Kit.""" + return renderer == "isaac_rtx" + + +def gallery_stage_paths() -> tuple[str, str]: + """Return the single-environment scene path and matching camera expression.""" + return "/World/envs/env_0/Scene", "/World/envs/env_.*/Scene/Camera" + + +def resolve_gallery_scene(scene: str) -> str: + """Preserve a Nucleus URI or resolve and validate a local scene path.""" + if scene.startswith("omniverse://"): + return scene + + scene_path = Path(scene).expanduser().resolve() + if not scene_path.is_file(): + raise ValueError(f"Scene does not exist: {scene_path}") + return str(scene_path) + + +def add_gallery_arguments(parser: argparse.ArgumentParser) -> None: + """Add renderer-gallery arguments without colliding with AppLauncher options.""" + script_dir = Path(__file__).resolve().parent + parser.add_argument("--renderer-backend", choices=tuple(_RENDERER_SLUGS), required=True) + parser.add_argument("--capture-group", choices=("standard", *_SIMPLE_SHADING_MODES), default="standard") + parser.add_argument("--newton-shadows", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--scene", type=resolve_gallery_scene, default=_DEFAULT_GALLERY_SCENE) + parser.add_argument( + "--output-dir", + type=Path, + default=script_dir.parents[2] / "docs" / "source" / "_static" / "overview" / "sensors", + ) + parser.add_argument("--width", type=int, default=640) + parser.add_argument("--height", type=int, default=360) + parser.add_argument("--frames", type=int, default=37) + parser.add_argument("--physics-steps-per-frame", type=int, default=3) + parser.add_argument("--warmup-steps", type=int, default=24) + + +def _parse_args() -> argparse.Namespace: + """Parse capture and Isaac Lab launcher arguments.""" + from isaaclab.app import AppLauncher + + parser = argparse.ArgumentParser(description=__doc__) + add_gallery_arguments(parser) + AppLauncher.add_app_launcher_args(parser) + parser.set_defaults(enable_cameras=True, headless=True) + args = parser.parse_args() + args.output_dir = args.output_dir.expanduser().resolve() + if args.width < 1 or args.height < 1: + parser.error("Image width and height must be positive.") + if args.frames < 6: + parser.error("At least six animation frames are required.") + if args.physics_steps_per_frame < 1: + parser.error("Physics steps per frame must be positive.") + if args.warmup_steps < 0: + parser.error("Warmup steps must be non-negative.") + try: + capture_data_types(args.renderer_backend, args.capture_group) + except ValueError as exc: + parser.error(str(exc)) + return args + + +def _make_renderer_cfg(renderer: str, *, enable_shadows: bool = True) -> Any: + """Create the selected renderer configuration.""" + if renderer == "newton": + from isaaclab_newton.renderers import NewtonWarpRendererCfg + + return NewtonWarpRendererCfg(enable_shadows=enable_shadows, enable_ambient_lighting=False) + if renderer == "ovrtx": + from isaaclab_ov.renderers import OVRTXRendererCfg + + return OVRTXRendererCfg() + if renderer == "isaac_rtx": + from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg + + return IsaacRtxRendererCfg(global_settings=IsaacRtxRendererGlobalSettingsCfg(ambient_light_intensity=0.0)) + raise ValueError(f"Unknown renderer: {renderer}") + + +def override_ovrtx_ambient_light(render_product_usd: str) -> str: + """Disable renderer-authored ambient fill while preserving the scene authored lights.""" + if _OVRTX_AMBIENT_LIGHT_SETTING not in render_product_usd: + raise RuntimeError("Expected the OVRTX render product to author its default ambient-light intensity.") + return render_product_usd.replace( + _OVRTX_AMBIENT_LIGHT_SETTING, + "float omni:rtx:rt:ambientLight:intensity = 0.0", + 1, + ) + + +@contextlib.contextmanager +def gallery_lighting_override(renderer: str) -> Iterator[None]: + """Apply capture-only renderer lighting overrides for the duration of camera creation.""" + if renderer != "ovrtx": + yield + return + + import isaaclab_ov.renderers.ovrtx_renderer as ovrtx_renderer + + original_builder = ovrtx_renderer.build_render_product_as_string + + def build_render_product_without_ambient_light(*args: Any, **kwargs: Any) -> tuple[str, str]: + render_product_usd, render_product_path = original_builder(*args, **kwargs) + return override_ovrtx_ambient_light(render_product_usd), render_product_path + + ovrtx_renderer.build_render_product_as_string = build_render_product_without_ambient_light + try: + yield + finally: + ovrtx_renderer.build_render_product_as_string = original_builder + + +def _create_camera_and_reset(renderer: str, create_camera: Callable[[], Any], reset_sim: Callable[[], None]) -> Any: + """Create the camera and initialize its renderer under capture lighting overrides.""" + with gallery_lighting_override(renderer): + camera = create_camera() + reset_sim() + return camera + + +def _capture(args: argparse.Namespace) -> None: + """Load the USD scene, render the selected outputs, and write documentation assets.""" + import numpy as np + import torch + from PIL import Image + + import isaaclab.sim as sim_utils + from isaaclab import cloner + from isaaclab.envs.utils.camera_colorizer import CameraFrameColorizer + from isaaclab.sensors import Camera, CameraCfg + + if renderer_requires_kit(args.renderer_backend): + sim_cfg = sim_utils.SimulationCfg(dt=1.0 / 60.0, render_interval=1, device=args.device, use_fabric=True) + else: + from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + + sim_utils.create_new_stage() + sim_cfg = sim_utils.SimulationCfg( + dt=1.0 / 60.0, + device=args.device, + physics=NewtonCfg(solver_cfg=MJWarpSolverCfg(integrator="implicitfast"), num_substeps=8), + ) + + data_types = capture_data_types(args.renderer_backend, args.capture_group) + sim = sim_utils.SimulationContext(sim_cfg) + scene_path, camera_path = gallery_stage_paths() + stage = sim_utils.get_current_stage() + stage.DefinePrim("/World/envs/env_0", "Xform") + scene_cfg = sim_utils.UsdFileCfg(usd_path=str(args.scene)) + scene_cfg.func(scene_path, scene_cfg) + env_positions = torch.zeros((1, 3), device=args.device) + clone_plan = cloner.clone_plan_from_env_0("/World/envs/env_0", "/World/envs/env_{}", 1, args.device, env_positions) + cloner.replicate(clone_plan, stage=stage) + camera = _create_camera_and_reset( + args.renderer_backend, + lambda: Camera( + CameraCfg( + prim_path=camera_path, + update_period=0.0, + width=args.width, + height=args.height, + data_types=list(data_types), + spawn=None, + renderer_cfg=_make_renderer_cfg(args.renderer_backend, enable_shadows=args.newton_shadows), + ) + ), + sim.reset, + ) + + def tensor_to_image(data: torch.Tensor, output_name: str) -> Image.Image: + data = snapshot_camera_tensor(data) + if output_name == "depth": + depth_min, depth_max = depth_display_bounds(data) + array = CameraFrameColorizer.colorize(data, "depth", depth_min=depth_min, depth_max=depth_max) + elif output_name == "normals": + array = CameraFrameColorizer.colorize(data, "normals") + elif output_name in {"semantic_segmentation", "instance_segmentation"}: + if data.ndim == 3 and data.shape[-1] >= 3 and data.dtype == torch.uint8: + array = data[..., :3].numpy() + else: + array = CameraFrameColorizer.colorize(data, "segmentation") + elif output_name == "motion_vectors": + return motion_vectors_to_image(data) + else: + array = data[..., :3].numpy() + if array.dtype != np.uint8: + array = np.clip(array, 0.0, 1.0) + array = (array * 255).astype(np.uint8) + return Image.fromarray(np.ascontiguousarray(array).copy(), mode="RGB") + + args.output_dir.mkdir(parents=True, exist_ok=True) + try: + for _ in range(args.warmup_steps): + camera.update(dt=0.0, force_recompute=True) + + thumbnail_index = thumbnail_frame_index(args.frames) + if args.capture_group == "standard": + rgb_frames: list[Image.Image] = [] + thumbnail_frames: dict[str, Image.Image] = {} + for frame_index in range(args.frames): + if frame_index > 0: + for _ in range(args.physics_steps_per_frame): + sim.step(render=False) + camera.update( + dt=sim.get_physics_dt() * args.physics_steps_per_frame, + force_recompute=True, + ) + if "rgb" in data_types: + rgb_frames.append(tensor_to_image(camera.data.output["rgb"].torch[0], "rgb")) + if frame_index == thumbnail_index: + for output_name in data_types: + if output_name != "rgb": + thumbnail_frames[output_name] = tensor_to_image( + camera.data.output[output_name].torch[0], output_name + ) + + rgb_path = args.output_dir / gallery_asset_name(args.renderer_backend, "rgb") + rgb_frames[0].save( + rgb_path, + save_all=True, + append_images=rgb_frames[1:], + duration=round(1000 / 12), + loop=0, + quality=86, + method=6, + ) + print(f"[INFO] Wrote {rgb_path}", flush=True) + for output_name, image in thumbnail_frames.items(): + output_path = args.output_dir / gallery_asset_name(args.renderer_backend, output_name) + image.save(output_path, optimize=True) + print(f"[INFO] Wrote {output_path}", flush=True) + else: + for _ in range(thumbnail_index * args.physics_steps_per_frame): + sim.step(render=False) + camera.update( + dt=sim.get_physics_dt() * args.physics_steps_per_frame, + force_recompute=True, + ) + output_name = data_types[0] + image = tensor_to_image(camera.data.output[output_name].torch[0], output_name) + output_path = args.output_dir / gallery_asset_name(args.renderer_backend, output_name) + image.save(output_path, optimize=True) + print(f"[INFO] Wrote {output_path}", flush=True) + finally: + sim.stop() + sim.clear_instance() + + +def main() -> None: + """Launch Isaac Sim and capture the selected renderer gallery group.""" + from isaaclab.app import AppLauncher + + args = _parse_args() + if not renderer_requires_kit(args.renderer_backend): + _capture(args) + else: + app_launcher = AppLauncher(args) + simulation_app = app_launcher.app + try: + _capture(args) + finally: + simulation_app.close() + + +if __name__ == "__main__": + main() diff --git a/tools/docs/media/generate_renderer_gallery.sh b/tools/docs/media/generate_renderer_gallery.sh new file mode 100755 index 000000000000..e438e22dda86 --- /dev/null +++ b/tools/docs/media/generate_renderer_gallery.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +set -euo pipefail + +if [[ "$#" -gt 1 ]]; then + echo "Usage: $0 [renderer-gallery-scene-path-or-uri]" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" +OUTPUT_DIR="${REPO_ROOT}/docs/source/_static/overview/sensors" +CAPTURE_SCRIPT="${SCRIPT_DIR}/capture_renderer_gallery.py" +SIMPLE_SHADING_MODES=( + simple_shading_constant_diffuse + simple_shading_diffuse_mdl + simple_shading_full_mdl +) +COMMON_ARGS=( + --output-dir "${OUTPUT_DIR}" + --width 640 + --height 360 + --frames 37 + --warmup-steps 24 +) + +if [[ "$#" -eq 1 ]]; then + COMMON_ARGS+=(--scene "$1") +fi + +if ! command -v uv >/dev/null 2>&1; then + echo "Error: uv is required to generate the renderer gallery." >&2 + exit 1 +fi + +if [[ "${OMNI_KIT_ACCEPT_EULA:-${ACCEPT_EULA:-}}" != "Y" ]]; then + echo "Error: set OMNI_KIT_ACCEPT_EULA=Y or ACCEPT_EULA=Y after reviewing NVIDIA's EULA." >&2 + exit 1 +fi + +export ACCEPT_EULA=Y +export OMNI_KIT_ACCEPT_EULA=Y + +cd "${REPO_ROOT}" +mkdir -p "${OUTPUT_DIR}" + +capture_kitless() { + local renderer="$1" + local capture_group="$2" + uv run --frozen --extra ovrtx python "${CAPTURE_SCRIPT}" \ + --renderer-backend "${renderer}" \ + --capture-group "${capture_group}" \ + "${COMMON_ARGS[@]}" +} + +capture_isaac_rtx() { + local capture_group="$1" + uv run --frozen --extra isaacsim python "${CAPTURE_SCRIPT}" \ + --renderer-backend isaac_rtx \ + --capture-group "${capture_group}" \ + "${COMMON_ARGS[@]}" +} + +capture_kitless newton standard +capture_kitless ovrtx standard +for mode in "${SIMPLE_SHADING_MODES[@]}"; do + capture_kitless ovrtx "${mode}" +done + +capture_isaac_rtx standard +for mode in "${SIMPLE_SHADING_MODES[@]}"; do + capture_isaac_rtx "${mode}" +done diff --git a/tools/docs/media/test_capture_renderer_gallery.py b/tools/docs/media/test_capture_renderer_gallery.py new file mode 100644 index 000000000000..5bb1dcad6150 --- /dev/null +++ b/tools/docs/media/test_capture_renderer_gallery.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import argparse + +import numpy as np +import pytest +import torch + +from tools.docs.media import capture_renderer_gallery +from tools.docs.media.capture_renderer_gallery import motion_vectors_to_image, thumbnail_frame_index + + +@pytest.mark.parametrize( + ("depth", "expected"), + [ + (torch.tensor([[float("nan"), 1.5, 4.0, float("inf")]]), (1.5, 4.0)), + (torch.tensor([[4.0, 20.0]]), (4.0, 20.0)), + ], +) +def test_depth_display_bounds_use_finite_frame_extents(depth, expected): + assert capture_renderer_gallery.depth_display_bounds(depth) == expected + + +def test_depth_display_bounds_reject_frame_with_no_finite_samples(): + depth = torch.tensor([[float("nan"), float("inf")]]) + + with pytest.raises(ValueError, match="finite"): + capture_renderer_gallery.depth_display_bounds(depth) + + +def test_gallery_capture_defaults_to_published_scene(): + parser = argparse.ArgumentParser() + capture_renderer_gallery.add_gallery_arguments(parser) + + args = parser.parse_args(["--renderer-backend", "newton"]) + + assert args.scene == ( + "omniverse://isaac-dev.ov.nvidia.com/Isaac/IsaacLab/Docs/Renderers/renderer_gallery_scene.usda" + ) + + +def test_gallery_scene_preserves_nucleus_uri(): + scene_uri = "omniverse://example.com/path/to/scene.usda" + + assert capture_renderer_gallery.resolve_gallery_scene(scene_uri) == scene_uri + + +def test_gallery_scene_resolves_existing_local_path(tmp_path, monkeypatch): + scene_path = tmp_path / "scene.usda" + scene_path.touch() + monkeypatch.chdir(tmp_path) + + assert capture_renderer_gallery.resolve_gallery_scene("scene.usda") == str(scene_path) + + +def test_gallery_scene_rejects_missing_local_path(tmp_path): + scene_path = tmp_path / "missing.usda" + + with pytest.raises(ValueError, match="Scene does not exist"): + capture_renderer_gallery.resolve_gallery_scene(str(scene_path)) + + +def test_thumbnail_uses_sixth_captured_frame(): + assert thumbnail_frame_index(6) == 5 + + +@pytest.mark.parametrize("frame_count", [0, 5]) +def test_thumbnail_requires_six_frames(frame_count): + with pytest.raises(ValueError, match="six"): + thumbnail_frame_index(frame_count) + + +def test_motion_vector_image_includes_direction_arrows(): + vectors = torch.zeros((64, 64, 4), dtype=torch.float32) + vectors[8:56, 8:56, 0] = 1.0 + vectors[8:56, 8:56, 1] = 0.5 + + image = motion_vectors_to_image(vectors) + + pixels = np.asarray(image) + assert pixels.shape == (64, 64, 3) + assert np.any(np.all(pixels == 255, axis=-1)) + + +def test_motion_vector_arrows_convert_v_axis_to_screen_y(): + vectors = torch.zeros((64, 64, 4), dtype=torch.float32) + vectors[32, 32, 1] = -1.0 + + pixels = np.asarray(motion_vectors_to_image(vectors)) + white_pixels = np.all(pixels == 255, axis=-1) + + assert np.any(white_pixels[33:, 28:37]) + assert not np.any(white_pixels[:32])