Fix progress score to be normalized (#1137)

## Summary
Fix our progress scoring to be normalized.

## Detailed description
- **Why:** `overall_score` was an un-normalized weighted sum over
objectives, not a [0, 1] progress fraction.
- **What:**
  - The progress tracker now normalized `overall_score`
- `report_data` reads it directly; the `max_score` helper and the
total-groups normalization it fed are removed.
- **Impact:**
- Backwards compatibility issue with existing benchmark results. These
are now not correctly normalized by consumers in the updated code.
  - Acceptable.

Signed-off-by: alex <amillane@nvidia.com>

Author: alexmillane

isaaclab_arena/progress_tracking/progress_tracker.py

@@ -74,7 +74,7 @@ class ProgressState:
     """Per-objective state, keyed by ProgressObjective name."""
 
     overall_score: float
-    """Sum of each objective's score weighted by ProgressObjective.score."""
+    """Sum of each objective's score weighted by ProgressObjective.score, normalized to [0, 1]."""
 
     all_complete: bool
     """Whether every objective is complete for this env."""
@@ -326,19 +326,25 @@ def get_state(self) -> list[ProgressState]:
         completeness = [runner.is_complete() for runner in self.runners]
         scores = [runner.overall_score_per_env() for runner in self.runners]
 
+        # Total objective weight for normalization.
+        total_objective_weight = sum(runner.progress_objective.score for runner in self.runners)
+
         output: list[ProgressState] = []
         for env_idx in range(self.num_envs):
             # Build a per-env state from each runner's state.
             progress_objective_states: dict[str, ProgressObjectiveState] = {}
-            overall_score = 0.0
+            weighted_score = 0.0
             all_complete = True
             for i, runner in enumerate(self.runners):
                 objective = runner.progress_objective
                 state = runner.get_state_for_env(env_idx, completeness[i][env_idx], scores[i][env_idx])
                 progress_objective_states[objective.name] = state
-                overall_score += objective.score * state.score
+                weighted_score += objective.score * state.score
                 all_complete = all_complete and state.is_complete
 
+            overall_score = (
+                max(0.0, min(1.0, weighted_score / total_objective_weight)) if total_objective_weight > 0 else 0.0
+            )
             output.append(
                 ProgressState(
                     progress_objectives=progress_objective_states,
@@ -385,7 +391,7 @@ class ProgressTrackingRecorder(RecorderTerm):
                         ),
                         ...
                     },
-                    overall_score=float,                   # weighted by ProgressObjective.score
+                    overall_score=float,                   # weighted mean of objective scores, in [0, 1]
                     all_complete=bool,
                 ),
                 ...

isaaclab_arena/tests/test_progress_objective_tracking.py

@@ -558,12 +558,17 @@ def _test_gating_sequential_task_end_to_end(simulation_app) -> bool:
         sm.step(env, step_index=env.episode_length_buf)
         assert sm.get_state()[0].progress_objectives["a"].is_complete
         assert not sm.get_state()[0].progress_objectives["b"].is_complete
+        # overall_score is the weighted mean of the two objectives (a=1.0, b=0.0) -> 0.5, not the
+        # un-normalized sum (1.0).
+        assert abs(sm.get_state()[0].overall_score - 0.5) < SCORE_TOL
 
         # Advances to subtask 1 so pred_b is now active.
         env._current_subtask_idx = [1]
         _advance_step(env)
         sm.step(env, step_index=env.episode_length_buf)
         assert sm.get_state()[0].progress_objectives["b"].is_complete
+        # Both objectives complete now -> normalized overall_score reaches 1.0.
+        assert abs(sm.get_state()[0].overall_score - 1.0) < SCORE_TOL
     except Exception as e:
         print(f"Error: {e}")
         traceback.print_exc()

isaaclab_arena/tests/test_report_data.py

@@ -59,17 +59,22 @@ def _write_run(experiment_dir, run_name: str, records: list[dict], cameras: tupl
     return run_dir
 
 
-def test_progress_fraction_normalizes_by_the_achievable_score():
-    episode = _episode({"progress": _progress({"a": 1, "b": 1, "c": 1}, [], score=1.5)})
+def test_progress_fraction_uses_recorded_overall_score():
+    # overall_score is recorded already normalized to [0, 1], so it is used directly.
+    episode = _episode({"progress": {"overall_score": 0.5}})
 
-    assert episode.max_score == 3.0
     assert episode.progress_fraction == 0.5
 
 
-def test_progress_fraction_is_none_without_recorded_objectives():
+def test_progress_fraction_clamps_out_of_range_overall_score():
+    episode = _episode({"progress": {"overall_score": 1.5}})
+
+    assert episode.progress_fraction == 1.0
+
+
+def test_progress_fraction_is_none_without_recorded_progress():
     episode = _episode({"success": True})
 
-    assert episode.max_score is None
     assert episode.progress_fraction is None
 
 

isaaclab_arena/visualization/report_data.py

@@ -66,18 +66,11 @@ def success(self) -> bool | None:
         success = self.record.get("success")
         return success if isinstance(success, bool) else None
 
-    @property
-    def max_score(self) -> float | None:
-        objectives = _progress_objectives(self.record)
-        total = sum(_as_float(objective.get("total_groups")) or 0.0 for objective in objectives.values())
-        return total if total > 0 else None
-
     @property
     def progress_fraction(self) -> float | None:
-        score, max_score = _as_float(_progress(self.record).get("overall_score")), self.max_score
-        if score is None or max_score is None:
-            return None
-        return max(0.0, min(1.0, score / max_score))
+        # overall_score is recorded already normalized to [0, 1] by the progress tracker.
+        score = _as_float(_progress(self.record).get("overall_score"))
+        return None if score is None else max(0.0, min(1.0, score))
 
     @property
     def all_objectives_complete(self) -> bool | None: