feat(model): add Nemotron 3.5 Super VL support

docs/fern/versions/nightly.yml

@@ -134,6 +134,8 @@ navigation:
       path: ./nightly/pages/models/nemotron/nemotron3-super.mdx
     - page: Nemotron 3.5 Lightning
       path: ./nightly/pages/models/nemotron/nemotron3.5-lightning.mdx
+    - page: Nemotron 3.5 Super VL
+      path: ./nightly/pages/models/nemotron/nemotron3.5-super-vl.mdx
     - page: Nemotron 3 Ultra
       path: ./nightly/pages/models/nemotron/nemotron3-ultra.mdx
     - page: Nemotron Nano V2 VL

docs/fern/versions/nightly/pages/models/README.mdx

@@ -4,7 +4,7 @@
 
 Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands.
 
-Current inventory: **26 model cards** and **369 concrete configurations**.
+Current inventory: **27 model cards** and **387 concrete configurations**.
 
 Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU.
 
@@ -112,7 +112,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
   <section class="verification-provider-card">
     <header class="verification-provider-heading">
       <strong>NVIDIA</strong>
-      <span>6 models</span>
+      <span>7 models</span>
     </header>
     <div class="verification-provider-models">
       <a class="verification-model-link" href="nemotron/nemotron-3-omni#verified-nemotron-3-nano-omni-30b-a3b-reasoning">
@@ -139,6 +139,10 @@ Choose a model to inspect its recorded import/export, training, and precision co
         <strong title="nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16">NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16</strong>
         <span class="verification-status verification-status--verified" title="Verified">✓ Verified</span>
       </a>
+      <a class="verification-model-link" href="nemotron/nemotron3.5-super-vl#verified-nemotron-3.5-super-vl-120b-a12b">
+        <strong title="nvidia/NVIDIA-Nemotron-3.5-Super-120B-A12B-SourceOfTruth">NVIDIA-Nemotron-3.5-Super-120B-A12B-SourceOfTruth</strong>
+        <span class="verification-status verification-status--partial" title="Partial">◐ Partial</span>
+      </a>
     </div>
   </section>
   <section class="verification-provider-card">

docs/models/README.md

@@ -4,7 +4,7 @@
 
 Choose a model to open its interactive import/export and training configurations. Every option comes directly from an authoritative YAML verification card; independent fields are never combined into synthetic commands.
 
-Current inventory: **26 model cards** and **369 concrete configurations**.
+Current inventory: **27 model cards** and **387 concrete configurations**.
 
 Each model page lets you select import/export, pretraining, benchmarking, SFT, LoRA, or long-context SFT and immediately see the exact command and expected result for the recorded precision and GPU.
 
@@ -112,7 +112,7 @@ Choose a model to inspect its recorded import/export, training, and precision co
   <section class="verification-provider-card">
     <header class="verification-provider-heading">
       <strong>NVIDIA</strong>
-      <span>6 models</span>
+      <span>7 models</span>
     </header>
     <div class="verification-provider-models">
       <a class="verification-model-link" href="nemotron/nemotron-3-omni.html#verified-nemotron-3-nano-omni-30b-a3b-reasoning">
@@ -139,6 +139,10 @@ Choose a model to inspect its recorded import/export, training, and precision co
         <strong title="nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16">NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16</strong>
         <span class="verification-status verification-status--verified" title="Verified">✓ Verified</span>
       </a>
+      <a class="verification-model-link" href="nemotron/nemotron3.5-super-vl.html#verified-nemotron-3.5-super-vl-120b-a12b">
+        <strong title="nvidia/NVIDIA-Nemotron-3.5-Super-120B-A12B-SourceOfTruth">NVIDIA-Nemotron-3.5-Super-120B-A12B-SourceOfTruth</strong>
+        <span class="verification-status verification-status--partial" title="Partial">◐ Partial</span>
+      </a>
     </div>
   </section>
   <section class="verification-provider-card">

examples/conversion/compare_hf_and_megatron/compare.py

@@ -109,6 +109,12 @@
 
 from megatron.bridge import AutoBridge
 from megatron.bridge.models.hf_pretrained.utils import is_safe_repo
+from megatron.bridge.models.nemotron_omni.inference_inputs import (
+    is_nemotron_omni,
+    load_nemotron_omni_video,
+    nemotron_omni_reference_metadata,
+    prepare_nemotron_omni_inputs,
+)
 from megatron.bridge.utils.common_utils import disable_mtp_for_inference, get_last_rank, print_rank_0
 from megatron.bridge.utils.safe_url import is_safe_public_http_url, safe_url_open
 
@@ -225,6 +231,8 @@ def is_vision_language_model(
             ),
             **_hf_revision_kwargs(revision),
         )
+        if is_nemotron_omni(config):
+            return True
 
         # Check for VL model indicators in config
         model_type = getattr(config, "model_type", "").lower()
@@ -280,6 +288,8 @@ def __init__(
         image_grid_thw=None,
         inference_context=None,
         mm_token_type_ids=None,
+        imgs_sizes=None,
+        num_frames=None,
     ):
         self.batch = dict(
             tokens=input_ids,
@@ -295,6 +305,10 @@ def __init__(
             self.batch["image_grid_thw"] = image_grid_thw
         if mm_token_type_ids is not None:
             self.batch["mm_token_type_ids"] = mm_token_type_ids
+        if imgs_sizes is not None:
+            self.batch["imgs_sizes"] = imgs_sizes
+        if num_frames is not None:
+            self.batch["num_frames"] = num_frames
 
         self._yielded = False
 
@@ -337,6 +351,9 @@ def vlm_forward_step(data_iterator, model, **kwargs) -> torch.Tensor:
         forward_args["image_grid_thw"] = batch["image_grid_thw"]
     if "mm_token_type_ids" in batch:
         forward_args["mm_token_type_ids"] = batch["mm_token_type_ids"]
+    for key in ("imgs_sizes", "num_frames"):
+        if key in batch:
+            forward_args[key] = batch[key]
 
     def loss_func(x, **kwargs):
         return x
@@ -516,6 +533,14 @@ def _load_hf_model(args, is_vl_model: bool):
         ),
         **_hf_revision_kwargs(args.hf_revision),
     }
+    if getattr(args, "disable_hf_video_pruning", False):
+        config = AutoConfig.from_pretrained(
+            args.hf_model_path,
+            trust_remote_code=load_kwargs["trust_remote_code"],
+            **_hf_revision_kwargs(args.hf_revision),
+        )
+        config.video_pruning_rate = 0.0
+        load_kwargs["config"] = config
     hf_model = model_class.from_pretrained(args.hf_model_path, **load_kwargs).to(args.hf_device).eval()
     print_rank_0(f"Loaded with {model_class.__name__}")
 
@@ -561,8 +586,13 @@ def _export_and_load_roundtrip_hf_model(args, is_vl_model: bool, megatron_model,
     if _is_rank_0():
         print_rank_0("Loading exported HF model for comparison...")
         model_class = get_model_class(args.model_class, is_vl_model)
+        config_kwargs = {}
+        if getattr(args, "disable_hf_video_pruning", False):
+            config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)
+            config.video_pruning_rate = 0.0
+            config_kwargs["config"] = config
         hf_model = (
-            model_class.from_pretrained(save_path, torch_dtype=torch.bfloat16, trust_remote_code=True)
+            model_class.from_pretrained(save_path, torch_dtype=torch.bfloat16, trust_remote_code=True, **config_kwargs)
             .to(args.hf_device)
             .eval()
         )
@@ -574,10 +604,10 @@ def _export_and_load_roundtrip_hf_model(args, is_vl_model: bool, megatron_model,
     return None
 
 
-def _get_hf_forward_model(hf_model, pixel_values):
+def _get_hf_forward_model(hf_model, pixel_values, pixel_values_videos=None):
     """Select a composite model's language backbone for text-only comparison."""
     language_model = getattr(hf_model, "language_model", None)
-    if pixel_values is None and isinstance(language_model, torch.nn.Module):
+    if pixel_values is None and pixel_values_videos is None and isinstance(language_model, torch.nn.Module):
         print_rank_0("Using the HuggingFace language backbone for a text-only comparison.")
         return language_model
     return hf_model
@@ -592,6 +622,7 @@ def _run_hf_inference(
     *,
     token_type_ids=None,
     mm_token_type_ids=None,
+    pixel_values_videos=None,
 ):
     """Run HuggingFace model inference and return results.
 
@@ -612,7 +643,7 @@ def _run_hf_inference(
     if not _is_rank_0() or hf_model is None:
         return None, None, None, None, None
 
-    hf_forward_model = _get_hf_forward_model(hf_model, pixel_values)
+    hf_forward_model = _get_hf_forward_model(hf_model, pixel_values, pixel_values_videos)
 
     input_device = input_ids.device
     try:
@@ -628,7 +659,13 @@ def _run_hf_inference(
             "attention_mask": torch.ones_like(input_ids, dtype=torch.bool).to(hf_device),
         }
         if pixel_values is not None:
-            hf_inputs["pixel_values"] = pixel_values.to(hf_device)
+            hf_inputs["pixel_values"] = (
+                [value.to(hf_device) for value in pixel_values]
+                if isinstance(pixel_values, (list, tuple))
+                else pixel_values.to(hf_device)
+            )
+        if pixel_values_videos is not None:
+            hf_inputs["pixel_values_videos"] = pixel_values_videos.to(hf_device)
         if image_grid_thw is not None:
             hf_inputs["image_grid_thw"] = image_grid_thw.to(hf_device)
         if token_type_ids is not None:
@@ -671,12 +708,14 @@ def _run_hf_inference(
         )
 
 
-def _load_hf_reference_logits(path, input_ids, tokenizer):
+def _load_hf_reference_logits(path, input_ids, tokenizer, *, expected_metadata=None):
     """Load rank-0 HF logits produced by a memory-bounded reference forward."""
     if not _is_rank_0():
         return None, None, None, None, None
 
     reference = torch.load(path, map_location="cpu", weights_only=True)
+    if expected_metadata is not None and reference.get("metadata") != expected_metadata:
+        raise ValueError("HF reference does not match the Omni source and native media inputs")
     reference_input_ids = reference.get("input_ids")
     if reference_input_ids is None or not torch.equal(reference_input_ids.cpu(), input_ids.cpu()):
         raise ValueError("HF reference logits were produced from different input token IDs")
@@ -874,6 +913,26 @@ def compare_models_one_step(args) -> None:
     # Detect model type
     is_vl_model = is_vision_language_model(args.hf_model_path, args.trust_remote_code, args.hf_revision)
     print_rank_0(f"Detected model type: {'Vision-Language' if is_vl_model else 'Text-only LLM'}")
+    config = AutoConfig.from_pretrained(
+        args.hf_model_path,
+        trust_remote_code=is_safe_repo(trust_remote_code=args.trust_remote_code, hf_path=args.hf_model_path),
+        **_hf_revision_kwargs(args.hf_revision),
+    )
+    is_omni = is_nemotron_omni(config)
+    video_path = getattr(args, "video_path", None)
+    disable_video_pruning = getattr(args, "disable_hf_video_pruning", False)
+    if video_path and (not is_omni or args.image_path):
+        raise ValueError("Video comparison currently requires Nemotron Omni and no simultaneous image input.")
+    if disable_video_pruning and not (is_omni and video_path):
+        raise ValueError("--disable-hf-video-pruning applies only to Omni video comparison.")
+    video_pruning_rate = None
+    if video_path:
+        video_pruning_rate = 0.0 if disable_video_pruning else float(getattr(config, "video_pruning_rate", 0.0))
+        if video_pruning_rate != 0.0:
+            raise ValueError(
+                "Bridge video is unpruned; explicitly use --disable-hf-video-pruning for a matched oracle."
+            )
+        print_rank_0("Comparing unpruned Omni video (HF video_pruning_rate=0); not original pruned-HF behavior.")
 
     # Validate vision requirements
     if args.image_path and not is_vl_model:
@@ -895,11 +954,37 @@ def compare_models_one_step(args) -> None:
     # Setup tokenizer and processor
     tokenizer, processor = _setup_tokenizer_and_processor(args, is_vl_model)
 
+    # Omni uses the native processor once, then adapts only the media layout for Bridge.
+    omni_inputs = None
+    reference_metadata = None
+    imgs_sizes = num_frames = hf_pixel_values_videos = None
     # Process inputs
     print_rank_0(f"Processing inputs - Prompt: '{args.prompt}', Image: {args.image_path}")
-    input_ids, pixel_values, image_grid_thw, token_type_ids, mm_token_type_ids = process_inputs(
-        tokenizer, processor, args.image_path, args.prompt, is_vl_model, args.tp
-    )
+    if is_omni and (args.image_path or video_path):
+        images = frames = metadata = None
+        if video_path:
+            frames, metadata = load_nemotron_omni_video(video_path, fps=args.video_fps)
+        else:
+            images = [load_image(args.image_path).convert("RGB")]
+        omni_inputs = prepare_nemotron_omni_inputs(
+            processor, prompt=args.prompt, images=images, video_frames=frames, video_metadata=metadata
+        )
+        reference_metadata = nemotron_omni_reference_metadata(
+            omni_inputs,
+            model=args.hf_model_path,
+            revision=args.hf_revision,
+            video_pruning_rate=video_pruning_rate,
+        )
+        input_ids = pad_input_ids_to_tp_multiple(omni_inputs.hf["input_ids"], args.tp, tokenizer.pad_token_id or 0)
+        pixel_values = omni_inputs.bridge["pixel_values"]
+        imgs_sizes = omni_inputs.bridge["imgs_sizes"]
+        num_frames = omni_inputs.bridge["num_frames"]
+        hf_pixel_values_videos = omni_inputs.hf.get("pixel_values_videos")
+        image_grid_thw = token_type_ids = mm_token_type_ids = None
+    else:
+        input_ids, pixel_values, image_grid_thw, token_type_ids, mm_token_type_ids = process_inputs(
+            tokenizer, processor, args.image_path, args.prompt, is_vl_model, args.tp
+        )
 
     # Move to GPU
     input_ids = input_ids.cuda()
@@ -911,24 +996,28 @@ def compare_models_one_step(args) -> None:
         token_type_ids = token_type_ids.cuda()
     if mm_token_type_ids is not None:
         mm_token_type_ids = mm_token_type_ids.cuda()
+    if imgs_sizes is not None:
+        imgs_sizes = imgs_sizes.cuda()
+        num_frames = num_frames.cuda()
 
     print_rank_0(f"Input shape: {input_ids.shape}")
     print_rank_0(f"Pixel values shape: {pixel_values.shape if pixel_values is not None else 'None'}")
 
     # Run HF model forward pass
     if args.hf_logits_path:
         hf_logits, hf_next_token, hf_logits_stats, hf_top5_info, logits_shape = _load_hf_reference_logits(
-            args.hf_logits_path, input_ids, tokenizer
+            args.hf_logits_path, input_ids, tokenizer, expected_metadata=reference_metadata
         )
     else:
         hf_logits, hf_next_token, hf_logits_stats, hf_top5_info, logits_shape = _run_hf_inference(
             hf_model,
             input_ids,
-            pixel_values,
+            omni_inputs.hf.get("pixel_values") if omni_inputs is not None else pixel_values,
             image_grid_thw,
             tokenizer,
             token_type_ids=token_type_ids,
             mm_token_type_ids=mm_token_type_ids,
+            pixel_values_videos=hf_pixel_values_videos,
         )
 
     del hf_model
@@ -975,6 +1064,8 @@ def compare_models_one_step(args) -> None:
                 pixel_values,
                 image_grid_thw,
                 mm_token_type_ids=mm_token_type_ids,
+                imgs_sizes=imgs_sizes,
+                num_frames=num_frames,
             )
             megatron_output = fwd_bwd_function(
                 forward_step_func=vlm_forward_step,
@@ -1095,6 +1186,13 @@ def build_parser() -> argparse.ArgumentParser:
         help="Path or URL to the image for vision-language generation (optional).",
     )
     parser.add_argument("--megatron_model_path", type=str, default=None, help="Path to the Megatron model checkpoint")
+    parser.add_argument("--video_path", default=None, help="Video path for Nemotron Omni native-processor comparison.")
+    parser.add_argument("--video-fps", type=float, default=2.0, help="Video frame sampling rate.")
+    parser.add_argument(
+        "--disable-hf-video-pruning",
+        action="store_true",
+        help="Explicitly configure an unpruned Omni HF video oracle matching Bridge; changes the HF reference only.",
+    )
     parser.add_argument(
         "--hf-logits-path",
         default=None,