Skip to content

Commit f6ee62a

Browse files
authored
[model] fix: map optional Step3.7 vision parameters (#5558)
Signed-off-by: Yu Yao <yaoyu.094@gmail.com>
1 parent 18c54d6 commit f6ee62a

2 files changed

Lines changed: 52 additions & 1 deletion

File tree

‎src/megatron/bridge/models/stepfun/step37_bridge.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,8 @@ def mapping_registry(self) -> MegatronMappingRegistry:
500500
vision_norm_param_mappings = {
501501
"vision_model.ln_pre.weight": "vision_model.ln_pre.weight",
502502
"vision_model.ln_pre.bias": "vision_model.ln_pre.bias",
503+
"vision_model.ln_post.weight": "vision_model.ln_post.weight",
504+
"vision_model.ln_post.bias": "vision_model.ln_post.bias",
503505
"vision_model.transformer.resblocks.*.ln_1.weight": "vision_model.transformer.resblocks.*.ln_1.weight",
504506
"vision_model.transformer.resblocks.*.ln_1.bias": "vision_model.transformer.resblocks.*.ln_1.bias",
505507
"vision_model.transformer.resblocks.*.ln_2.weight": "vision_model.transformer.resblocks.*.ln_2.weight",
@@ -519,6 +521,7 @@ def mapping_registry(self) -> MegatronMappingRegistry:
519521
"vision_model.vit_downsampler2.bias": "vision_model.vit_downsampler2.bias",
520522
# Top-level nn.Parameter on Step37VisionModel — AutoMapping
521523
# would see the top-level vision class as the owning module.
524+
"vision_model.class_embedding": "vision_model.class_embedding",
522525
"vision_model.positional_embedding": "vision_model.positional_embedding",
523526
# Attention — in_proj_* are bare nn.Parameter on
524527
# EncoderVisionAttention; out_proj.* are plain nn.Linear.

‎tests/unit_tests/models/stepfun/test_step37_bridge.py‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@
2121

2222
from megatron.bridge.models.conversion.utils import conform_config_to_reference
2323
from megatron.bridge.models.stepfun import step37_bridge as _step37_bridge_mod
24-
from megatron.bridge.models.stepfun.configuration_step37 import Step37Config
24+
from megatron.bridge.models.stepfun.configuration_step37 import Step37Config, Step37VisionConfig
2525
from megatron.bridge.models.stepfun.modelling_step37 import transformer_block as _step37_block_mod
2626
from megatron.bridge.models.stepfun.modelling_step37.image_insert_embedding import ImageInsertEmbedding
2727
from megatron.bridge.models.stepfun.modelling_step37.transformer_block import get_step37_text_layer_spec
28+
from megatron.bridge.models.stepfun.modelling_step37.vision_model import Step37VisionModel
2829
from megatron.bridge.models.stepfun.step35_bridge import build_step35_layer_spec
2930
from megatron.bridge.models.stepfun.step37_bridge import Step37Bridge
3031

@@ -121,6 +122,53 @@ def test_projector_bias_schema_roundtrips(self):
121122
) == ("vit_large_projector.bias", True)
122123

123124

125+
class TestStep37VisionMappings:
126+
def test_optional_vision_parameters_have_checkpoint_mappings(self):
127+
vision_config = Step37VisionConfig(
128+
width=8,
129+
layers=0,
130+
heads=1,
131+
image_size=14,
132+
patch_size=14,
133+
mlp_ratio=1,
134+
use_cls_token=True,
135+
use_ln_pre=False,
136+
use_ln_post=True,
137+
)
138+
vision_model = Step37VisionModel(vision_config)
139+
parameter_names = set(dict(vision_model.named_parameters()))
140+
assert {"class_embedding", "ln_post.weight", "ln_post.bias"} <= parameter_names
141+
142+
disabled_model = Step37VisionModel(
143+
Step37VisionConfig(
144+
width=8,
145+
layers=0,
146+
heads=1,
147+
image_size=14,
148+
patch_size=14,
149+
mlp_ratio=1,
150+
use_cls_token=False,
151+
use_ln_pre=False,
152+
use_ln_post=False,
153+
)
154+
)
155+
disabled_parameter_names = set(dict(disabled_model.named_parameters()))
156+
assert "class_embedding" not in disabled_parameter_names
157+
assert "ln_post.weight" not in disabled_parameter_names
158+
assert "ln_post.bias" not in disabled_parameter_names
159+
160+
bridge = Step37Bridge()
161+
bridge.hf_config = SimpleNamespace(
162+
text_config=SimpleNamespace(num_hidden_layers=0, num_nextn_predict_layers=0)
163+
)
164+
registry = bridge.mapping_registry()
165+
for parameter_name in ("class_embedding", "ln_post.weight", "ln_post.bias"):
166+
megatron_name = f"vision_model.{parameter_name}"
167+
mapping = registry.megatron_to_hf_lookup(megatron_name)
168+
assert mapping is not None
169+
assert mapping.hf_param == megatron_name
170+
171+
124172
class TestGetStep37TextLayerSpec:
125173
def test_delegates_to_step35_builder(self):
126174
"""Step3.7's text decoder spec is Step-3.5's hybrid layer spec, passed through verbatim."""

0 commit comments

Comments
 (0)