isaac-sim
diff --git a/‎source/isaaclab/changelog.d/fix-semantic-segmentation-dtype.rst‎
Lines changed: 8 additions & 0 deletions b/‎source/isaaclab/changelog.d/fix-semantic-segmentation-dtype.rst‎
Lines changed: 8 additions & 0 deletions
diff --git a/‎source/isaaclab/isaaclab/utils/images.py‎
Lines changed: 10 additions & 3 deletions b/‎source/isaaclab/isaaclab/utils/images.py‎
Lines changed: 10 additions & 3 deletions
diff --git a/‎source/isaaclab/test/utils/test_images.py‎
Lines changed: 18 additions & 3 deletions b/‎source/isaaclab/test/utils/test_images.py‎
Lines changed: 18 additions & 3 deletions
diff --git a/‎source/isaaclab_tasks/changelog.d/fix-cartpole-semantic-segmentation-dtype.rst‎
Lines changed: 10 additions & 0 deletions b/‎source/isaaclab_tasks/changelog.d/fix-cartpole-semantic-segmentation-dtype.rst‎
Lines changed: 10 additions & 0 deletions
diff --git a/‎source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py‎
Lines changed: 7 additions & 3 deletions b/‎source/isaaclab_tasks/isaaclab_tasks/core/cartpole/cartpole_direct_camera_env.py‎
Lines changed: 7 additions & 3 deletions
diff --git a/‎source/isaaclab_tasks/isaaclab_tasks/core/cartpole/mdp/observations.py‎
Lines changed: 5 additions & 2 deletions b/‎source/isaaclab_tasks/isaaclab_tasks/core/cartpole/mdp/observations.py‎
Lines changed: 5 additions & 2 deletions
diff --git a/‎source/isaaclab_tasks/test/core/test_cartpole_camera_observations.py‎
Lines changed: 72 additions & 0 deletions b/‎source/isaaclab_tasks/test/core/test_cartpole_camera_observations.py‎
Lines changed: 72 additions & 0 deletions
@@ -0,0 +1,8 @@
+Fixed
+^^^^^
+
+* Fixed :func:`isaaclab.utils.images.normalize_camera_image` returning non-colorized
+  ``"semantic_segmentation"`` unchanged. Such output is an ``int32`` label map on every renderer,
+  and feeding it to a convolution raised ``Input type (int) and bias type (float) should be the
+  same``. It is now cast to ``float32``; label ids carry no scale, so they are not rescaled.
+  Colorized (``uint8`` RGBA) segmentation keeps its existing ``(x / 255) - mean`` normalization.
@@ -52,6 +52,8 @@ def normalize_camera_image(
 
     Dispatch (in order of check):
 
+    - non-colorized ``"semantic_segmentation"`` (any non-``uint8`` dtype, ``int32`` label ids in
+      practice): cast to ``float32``. Label ids carry no scale, so no further normalization.
     - :func:`is_rgb_like` or colorized ``"semantic_segmentation"`` (``uint8``) and contiguous
       4D: routes to the
       fused Warp kernel via :func:`~isaaclab.utils.warp.ops.normalize_image_uint8`. ``out`` and
@@ -79,10 +81,15 @@ def normalize_camera_image(
 
     Returns:
         The normalized tensor. For RGB-like and colorized semantic-segmentation input this is a
-        fresh (or pre-allocated) float32 tensor; for depth-like input it is ``images`` itself
-        (mutated in place); for normals-like input it is a new tensor; for anything else,
-        ``images`` unchanged.
+        fresh (or pre-allocated) float32 tensor; for non-colorized semantic segmentation it is
+        ``images`` cast to float32; for depth-like input it is ``images`` itself (mutated in
+        place); for normals-like input it is a new tensor; for anything else, ``images``
+        unchanged.
     """
+    if data_type == "semantic_segmentation" and images.dtype != torch.uint8:
+        # Non-colorized segmentation is an integer label map (``int32`` for every renderer).
+        # Label ids carry no scale, so cast for the downstream convolutions without rescaling.
+        return images.float()
     if is_rgb_like(data_type) or (data_type == "semantic_segmentation" and images.dtype == torch.uint8):
         if images.dtype == torch.uint8 and images.ndim == 4 and images.is_contiguous():
             return normalize_image_uint8(images, channel_dim=channel_dim, out=out)
 
@@ -146,8 +146,8 @@ def test_bchw_float_input_takes_pytorch_fallback(self, device):
         torch.testing.assert_close(out, expected)
 
 
-class TestNormalizeCameraImageColorizedSegmentation:
-    """Colorized segmentation dispatch."""
+class TestNormalizeCameraImageSegmentation:
+    """Segmentation dispatch, keyed on the tensor dtype rather than on the ``colorize`` flag."""
 
     def test_colorized_semantic_segmentation_is_normalized(self, device):
         """RGBA uint8 semantic segmentation produces a float32 normalized image."""
@@ -162,6 +162,21 @@ def test_colorized_semantic_segmentation_is_normalized(self, device):
         torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
         assert out.dtype == torch.float32
 
+    def test_non_colorized_semantic_segmentation_is_cast_to_float(self, device):
+        """int32 label-id segmentation is cast to float32 with the ids left untouched.
+
+        Non-colorized segmentation is ``int32`` for every renderer, and feeding it to a
+        convolution raises ``Input type (int) and bias type (float) should be the same``.
+        """
+        from isaaclab.utils.images import normalize_camera_image
+
+        torch.manual_seed(0)
+        src = torch.randint(0, 4, (2, 8, 8, 1), dtype=torch.int32, device=device)
+        out = normalize_camera_image(src, "semantic_segmentation")
+
+        assert out.dtype == torch.float32
+        torch.testing.assert_close(out, src.to(torch.float32))
+
 
 class TestNormalizeCameraImageDepth:
     """Depth-like dispatch: in-place ``inf -> 0``."""
@@ -192,7 +207,7 @@ def test_range_remap(self, device):
 class TestNormalizeCameraImagePassthrough:
     """Unknown data_types return the input unchanged."""
 
-    @pytest.mark.parametrize("data_type", ["semantic_segmentation", "instance_segmentation", "motion_vectors"])
+    @pytest.mark.parametrize("data_type", ["instance_segmentation", "motion_vectors"])
     def test_unknown_type_passthrough(self, device, data_type):
         from isaaclab.utils.images import normalize_camera_image
 
 
@@ -0,0 +1,10 @@
+Fixed
+^^^^^
+
+* Fixed the Cartpole camera tasks crashing at the first training step with
+  ``RuntimeError: Input type (unsigned char) and bias type (float) should be the same`` when the
+  ``semantic_segmentation`` preset was selected. Both the manager-based observation term and the
+  direct environment normalized only RGB-like and depth output, so segmentation reached the feature
+  extractor as an integer tensor. Segmentation is now routed through
+  :func:`isaaclab.utils.images.normalize_camera_image`, which keys on the tensor dtype and therefore
+  handles both colorized (``uint8`` RGBA) and non-colorized (``int32`` label ids) output.
@@ -8,6 +8,8 @@
 from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
+import torch
+
 import isaaclab.sim as sim_utils
 from isaaclab import cloner
 from isaaclab.assets import Articulation
@@ -78,15 +80,17 @@ def _get_observations(self) -> dict:
         camera_data = self._tiled_camera.data.output[data_type]
 
         rgb_like = is_rgb_like(data_type)
+        segmentation = data_type == "semantic_segmentation"
         # Defer normalize past the ring buffer when stacking RGB-like data so the ring holds
         # uint8 (4x cheaper per-step copies). Math is identical -- K frames live in disjoint
-        # channel slices of (B, K*C, H, W).
-        defer_normalize = self._stack is not None and rgb_like
+        # channel slices of (B, K*C, H, W). Colorized segmentation is uint8 RGBA and qualifies;
+        # non-colorized segmentation is an int32 label map and does not.
+        defer_normalize = self._stack is not None and (rgb_like or (segmentation and camera_data.dtype == torch.uint8))
 
         if data_type == "albedo":
             # albedo carries an extra alpha channel that the policy does not use
             camera_data = camera_data[..., :3]
-        if rgb_like and not defer_normalize:
+        if (rgb_like or segmentation) and not defer_normalize:
             camera_data = normalize_camera_image(camera_data, data_type)
         elif data_type == "depth":
             camera_data[camera_data == float("inf")] = 0
 
@@ -46,10 +46,13 @@ def __call__(self, env: ManagerBasedRLEnv, sensor_cfg: SceneEntityCfg, data_type
         camera_data = camera.data.output[data_type]
 
         rgb_like = is_rgb_like(data_type)
-        defer_normalize = self._stack is not None and rgb_like
+        segmentation = data_type == "semantic_segmentation"
+        # Colorized segmentation is uint8 RGBA, so it can ride the same deferred-normalize path as
+        # the RGB-like types; non-colorized segmentation is an int32 label map that cannot.
+        defer_normalize = self._stack is not None and (rgb_like or (segmentation and camera_data.dtype == torch.uint8))
         if data_type == "albedo":
             camera_data = camera_data[..., :3]
-        if rgb_like and not defer_normalize:
+        if (rgb_like or segmentation) and not defer_normalize:
             camera_data = normalize_camera_image(camera_data, data_type)
         elif data_type == "depth":
             camera_data[camera_data == float("inf")] = 0
 
@@ -0,0 +1,72 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Sim-free tests for the Cartpole camera observation term.
+
+Segmentation output is ``uint8`` RGBA when colorized and ``int32`` label ids when not. Either
+integer dtype crashes the feature extractor's first convolution, so the term must return float32
+for both. ``frame_stack`` is parametrized because the stacking path defers normalization.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from isaaclab.managers import ObservationTermCfg, SceneEntityCfg
+
+from isaaclab_tasks.core.cartpole.mdp.observations import CameraImageStack
+
+pytestmark = pytest.mark.unit
+
+
+@pytest.fixture(params=["cpu", "cuda:0"] if torch.cuda.is_available() else ["cpu"])
+def device(request):
+    return request.param
+
+
+def _observe(images: torch.Tensor, frame_stack: int, device: str) -> torch.Tensor:
+    """Run the observation term over ``images`` using a minimal environment stub."""
+    camera = SimpleNamespace(data=SimpleNamespace(output={"semantic_segmentation": images}))
+    env = SimpleNamespace(
+        cfg=SimpleNamespace(frame_stack=frame_stack),
+        num_envs=images.shape[0],
+        device=device,
+        scene=SimpleNamespace(sensors={"tiled_camera": camera}),
+    )
+    term = CameraImageStack(ObservationTermCfg(func=CameraImageStack), env)
+    return term(env, SceneEntityCfg("tiled_camera"), "semantic_segmentation")
+
+
+def _to_expected_layout(images: torch.Tensor, frame_stack: int) -> torch.Tensor:
+    """Convert BHWC to the channel-first layout, repeated as the ring buffer fills on first append."""
+    return images.permute(0, 3, 1, 2).repeat(1, frame_stack, 1, 1)
+
+
+@pytest.mark.parametrize("frame_stack", [1, 2])
+def test_colorized_segmentation_is_normalized_like_rgb(device, frame_stack):
+    """Colorized uint8 RGBA segmentation gets the same ``(x / 255) - per-image mean`` as RGB."""
+    torch.manual_seed(0)
+    images = torch.randint(0, 255, (2, 8, 8, 4), dtype=torch.uint8, device=device)
+
+    observation = _observe(images, frame_stack, device)
+
+    expected = images.float() / 255.0
+    expected = expected - torch.mean(expected, dim=(1, 2), keepdim=True)
+    assert observation.dtype == torch.float32
+    torch.testing.assert_close(observation, _to_expected_layout(expected, frame_stack), atol=1e-5, rtol=1e-5)
+
+
+@pytest.mark.parametrize("frame_stack", [1, 2])
+def test_non_colorized_segmentation_is_cast_to_float(device, frame_stack):
+    """Non-colorized int32 label ids are cast to float32 and, carrying no scale, left unrescaled."""
+    images = torch.arange(2 * 8 * 8, dtype=torch.int32, device=device).reshape(2, 8, 8, 1) % 5
+
+    observation = _observe(images, frame_stack, device)
+
+    assert observation.dtype == torch.float32
+    torch.testing.assert_close(observation, _to_expected_layout(images.float(), frame_stack))