|
| 1 | +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Native HF processor compatibility for the audio-free Super-VL checkpoint.""" |
| 16 | + |
| 17 | +import transformers |
| 18 | +from packaging.version import Version |
| 19 | +from transformers import AutoModelForImageTextToText, AutoProcessor, ProcessorMixin |
| 20 | + |
| 21 | +from megatron.bridge.models.hf_pretrained.causal_lm import PreTrainedCausalLM |
| 22 | + |
| 23 | + |
| 24 | +class Nemotron35VLPretrained(PreTrainedCausalLM): |
| 25 | + """Keep native processor compatibility separate from legacy remote-code loading.""" |
| 26 | + |
| 27 | + def _uses_remote_model(self) -> bool: |
| 28 | + """Check the checkpoint registration, not only permission to execute it.""" |
| 29 | + return self.trust_remote_code and bool( |
| 30 | + (getattr(self.config, "auto_map", None) or {}).get("AutoModelForCausalLM") |
| 31 | + ) |
| 32 | + |
| 33 | + def _get_auto_model_class(self) -> type: |
| 34 | + """Use native VL registration while retaining the remote CausalLM mapping.""" |
| 35 | + if self._text_only: |
| 36 | + return super()._get_auto_model_class() |
| 37 | + # Permission to execute remote code does not imply that the checkpoint |
| 38 | + # uses it: native exports have no remote CausalLM registration. |
| 39 | + if self._uses_remote_model(): |
| 40 | + return super()._get_auto_model_class() |
| 41 | + if Version(transformers.__version__) < Version("5.19.0"): |
| 42 | + raise ValueError("Native Super-VL requires Transformers >=5.19.0; use trust_remote_code=True otherwise.") |
| 43 | + return AutoModelForImageTextToText |
| 44 | + |
| 45 | + def _load_processor(self) -> ProcessorMixin | None: |
| 46 | + if self.model_name_or_path is None or self._uses_remote_model(): |
| 47 | + return super()._load_processor() |
| 48 | + |
| 49 | + version = Version(transformers.__version__) |
| 50 | + if version < Version("5.19.0"): |
| 51 | + raise ValueError("Native Super-VL requires Transformers >=5.19.0; use trust_remote_code=True otherwise.") |
| 52 | + |
| 53 | + processor_cls = AutoProcessor |
| 54 | + # Bound the workaround to the release whose loading behavior we tested. |
| 55 | + # Future releases use their own loader and surface errors rather than |
| 56 | + # silently dropping a required VLM artifact during conversion/export. |
| 57 | + if version.base_version == "5.19.0" and getattr(self.config, "audio_config", None) is None: |
| 58 | + from megatron.bridge.models.nemotron_omni.processing_super_vl import Nemotron35VLProcessor |
| 59 | + |
| 60 | + processor_cls = Nemotron35VLProcessor |
| 61 | + return processor_cls.from_pretrained(self.model_name_or_path, trust_remote_code=False, **self.init_kwargs) |
0 commit comments