Skip to content

Commit 998128a

Browse files
authored
[ckpt] fix: Save Energon state through MSC (#5546)
Signed-off-by: Yu Yao <yaoyu.094@gmail.com>
1 parent 8867945 commit 998128a

2 files changed

Lines changed: 36 additions & 1 deletion

File tree

‎src/megatron/bridge/training/checkpointing.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1765,7 +1765,11 @@ def maybe_save_dataloader_state(
17651765

17661766
dataloader_save_dict = {}
17671767
dataloader_save_dict["dataloader_state_dict"] = train_dataloader_state_dict
1768-
torch.save(dataloader_save_dict, data_state_save_path)
1768+
if MultiStorageClientFeature.is_enabled():
1769+
msc = MultiStorageClientFeature.import_package()
1770+
msc.torch.save(dataloader_save_dict, data_state_save_path)
1771+
else:
1772+
torch.save(dataloader_save_dict, data_state_save_path)
17691773

17701774

17711775
def maybe_load_dataloader_state(

‎tests/unit_tests/training/test_checkpointing.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5340,3 +5340,34 @@ def test_writes_to_msc_path(self, tmp_path):
53405340
assert msc.torch.load(state_path, weights_only=True) == {
53415341
"dataloader_state_dict": {"dummy_energon_state": "xyz"}
53425342
}
5343+
5344+
@patch("megatron.bridge.training.checkpointing.torch.save")
5345+
@patch("megatron.bridge.training.checkpointing.ensure_directory_exists")
5346+
@patch("megatron.bridge.training.checkpointing.get_checkpoint_name")
5347+
def test_writes_nondefault_msc_uri_through_remote_adapter(
5348+
self, mock_get_checkpoint_name, mock_ensure_directory, mock_torch_save
5349+
):
5350+
"""A named MSC profile must not be passed to PyTorch's local-path serializer."""
5351+
mock_get_checkpoint_name.return_value = "msc://named/checkpoints/energon/iter_0000010"
5352+
msc = Mock()
5353+
train_iterator = self._iterator()
5354+
5355+
with (
5356+
patch.object(MultiStorageClientFeature, "is_enabled", return_value=True),
5357+
patch.object(MultiStorageClientFeature, "import_package", return_value=msc),
5358+
patch("megatron.bridge.training.checkpointing.torch.distributed.barrier"),
5359+
):
5360+
maybe_save_dataloader_state(
5361+
Mock(),
5362+
train_iterator,
5363+
10,
5364+
"msc://named/checkpoints/energon",
5365+
pg_collection=self._pg(),
5366+
)
5367+
5368+
expected_path = "msc://named/checkpoints/energon/iter_0000010/train_dataloader_dprank000.pt"
5369+
mock_ensure_directory.assert_called_once_with(expected_path)
5370+
msc.torch.save.assert_called_once_with(
5371+
{"dataloader_state_dict": {"dummy_energon_state": "xyz"}}, expected_path
5372+
)
5373+
mock_torch_save.assert_not_called()

0 commit comments

Comments
 (0)