Skip to content

Commit 85cda92

Browse files
authored
Integrate MXFP8 weights into MFSDP (#7114)
Signed-off-by: Jingyue Wu <jingyuew@nvidia.com>
1 parent cc250b9 commit 85cda92

6 files changed

Lines changed: 328 additions & 142 deletions

File tree

‎megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/dbuffer.py‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ def distribute_tensors(
294294
A DBuffer whose real local storage matches ``placements``. Ranges
295295
corresponding to meta tensors are left uninitialized.
296296
"""
297-
tensors = tuple(tensor.detach().contiguous() for tensor in tensors)
297+
tensors = tuple(tensors)
298298
if not tensors:
299299
raise ValueError("DBuffer.distribute_tensors() requires at least one tensor.")
300300

@@ -312,21 +312,27 @@ def distribute_tensors(
312312
device=mesh.device_type,
313313
block_size=block_size,
314314
)
315-
# Only logical tensor ranges are initialized. Padding and layout gaps are not
316-
# observable through get_tensor_view() and can remain unspecified.
317315
for index, tensor in enumerate(tensors):
318-
owned_range = buffer._get_owned_range(index)
319-
if owned_range is None or tensor.is_meta:
320-
continue
321-
322-
source_slice = tensor.view(-1).narrow(
323-
0, owned_range.tensor_relative_offset, owned_range.numel
324-
)
325-
buffer.local_buffer.narrow(
326-
0, owned_range.buffer_relative_offset, owned_range.numel
327-
).copy_(source_slice)
316+
buffer.copy_from(index, tensor)
328317
return buffer
329318

319+
def copy_from(self, index: int, tensor: torch.Tensor) -> None:
320+
"""Copy a full logical tensor's local owned range into this buffer.
321+
322+
Meta tensors leave their owned range unspecified. Padding and layout gaps
323+
are not observable through ``get_tensor_view()`` and remain unspecified.
324+
"""
325+
owned_range = self._get_owned_range(index)
326+
if owned_range is None or tensor.is_meta:
327+
return
328+
tensor = tensor.detach().contiguous()
329+
source_slice = tensor.view(-1).narrow(
330+
0, owned_range.tensor_relative_offset, owned_range.numel
331+
)
332+
self.local_buffer.narrow(0, owned_range.buffer_relative_offset, owned_range.numel).copy_(
333+
source_slice
334+
)
335+
330336
def _create_or_validate_out(
331337
self,
332338
out: "DBuffer | None",

‎megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/module.py‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@
3030
from .countdown import Countdown
3131
from .indexed_order import IndexedOrder
3232
from .module_utils import get_parameter_owner
33-
from .parameter_group import FsdpParameterGroup, get_containing_parameter_group
34-
from .placement import Flat
33+
from .parameter_group import FsdpParameterGroup, effective_dtype, get_containing_parameter_group
34+
from .placement import BlockAtomic, Flat
3535
from .schedule import SchedulePolicy
3636

3737

@@ -213,7 +213,8 @@ def __init__(
213213
raise ValueError(f"grad_divisor must be positive, got {grad_divisor}.")
214214
parameter_groups = []
215215
for group_parameters in _group_parameters(owned_parameters):
216-
group_dtype = next(iter(group_parameters.values())).dtype
216+
first_parameter = next(iter(group_parameters.values()))
217+
group_dtype = effective_dtype(first_parameter)
217218
parameter_groups.append(
218219
FsdpParameterGroup(
219220
owning_module=self,
@@ -619,25 +620,27 @@ def visit(submodule: nn.Module, submodule_fqn: str) -> None:
619620
def _group_parameters(parameters: dict[str, nn.Parameter]) -> list[dict[str, nn.Parameter]]:
620621
grouped: dict[tuple[torch.dtype, bool], dict[str, nn.Parameter]] = {}
621622
for name, parameter in parameters.items():
622-
key = (parameter.dtype, parameter.requires_grad)
623+
key = (effective_dtype(parameter), parameter.requires_grad)
623624
grouped.setdefault(key, {})[name] = parameter
624625
return [grouped[key] for key in grouped]
625626

626627

627628
def _specialize_placements(
628-
placements: tuple[Placement, ...], dtype: torch.dtype
629+
placements: tuple[Placement, ...], group_dtype: torch.dtype
629630
) -> tuple[Placement, ...]:
630631
"""Specialize public placements for one homogeneous parameter group.
631632
632-
Today every parameter group maps Torch's user-facing ``Shard(0)`` to the
633-
DBuffer-specific ``Flat`` format. This dtype-homogeneous group boundary is
634-
where MXFP8 groups will instead select ``BlockAtomic``.
633+
Map Torch's user-facing ``Shard(0)`` to ``BlockAtomic`` for MXFP8 groups
634+
and ``Flat`` for ordinary floating-point groups.
635635
"""
636-
if dtype not in (torch.float32, torch.bfloat16, torch.float16):
637-
raise NotImplementedError(f"Unsupported dtype: {dtype}.")
636+
if group_dtype not in (torch.uint8, torch.float32, torch.bfloat16, torch.float16):
637+
raise NotImplementedError(f"Unsupported group dtype: {group_dtype}.")
638638
for placement in placements:
639639
if type(placement) is Shard and placement.dim != 0:
640640
raise NotImplementedError(
641641
"MFSDP currently supports only dim-0 Shard placements, " f"got {placement!r}."
642642
)
643-
return tuple(Flat() if type(placement) is Shard else placement for placement in placements)
643+
placement_type = BlockAtomic(32) if group_dtype == torch.uint8 else Flat()
644+
return tuple(
645+
placement_type if type(placement) is Shard else placement for placement in placements
646+
)

0 commit comments

Comments
 (0)