|
30 | 30 | from .countdown import Countdown |
31 | 31 | from .indexed_order import IndexedOrder |
32 | 32 | from .module_utils import get_parameter_owner |
33 | | -from .parameter_group import FsdpParameterGroup, get_containing_parameter_group |
34 | | -from .placement import Flat |
| 33 | +from .parameter_group import FsdpParameterGroup, effective_dtype, get_containing_parameter_group |
| 34 | +from .placement import BlockAtomic, Flat |
35 | 35 | from .schedule import SchedulePolicy |
36 | 36 |
|
37 | 37 |
|
@@ -213,7 +213,8 @@ def __init__( |
213 | 213 | raise ValueError(f"grad_divisor must be positive, got {grad_divisor}.") |
214 | 214 | parameter_groups = [] |
215 | 215 | for group_parameters in _group_parameters(owned_parameters): |
216 | | - group_dtype = next(iter(group_parameters.values())).dtype |
| 216 | + first_parameter = next(iter(group_parameters.values())) |
| 217 | + group_dtype = effective_dtype(first_parameter) |
217 | 218 | parameter_groups.append( |
218 | 219 | FsdpParameterGroup( |
219 | 220 | owning_module=self, |
@@ -619,25 +620,27 @@ def visit(submodule: nn.Module, submodule_fqn: str) -> None: |
619 | 620 | def _group_parameters(parameters: dict[str, nn.Parameter]) -> list[dict[str, nn.Parameter]]: |
620 | 621 | grouped: dict[tuple[torch.dtype, bool], dict[str, nn.Parameter]] = {} |
621 | 622 | for name, parameter in parameters.items(): |
622 | | - key = (parameter.dtype, parameter.requires_grad) |
| 623 | + key = (effective_dtype(parameter), parameter.requires_grad) |
623 | 624 | grouped.setdefault(key, {})[name] = parameter |
624 | 625 | return [grouped[key] for key in grouped] |
625 | 626 |
|
626 | 627 |
|
627 | 628 | def _specialize_placements( |
628 | | - placements: tuple[Placement, ...], dtype: torch.dtype |
| 629 | + placements: tuple[Placement, ...], group_dtype: torch.dtype |
629 | 630 | ) -> tuple[Placement, ...]: |
630 | 631 | """Specialize public placements for one homogeneous parameter group. |
631 | 632 |
|
632 | | - Today every parameter group maps Torch's user-facing ``Shard(0)`` to the |
633 | | - DBuffer-specific ``Flat`` format. This dtype-homogeneous group boundary is |
634 | | - where MXFP8 groups will instead select ``BlockAtomic``. |
| 633 | + Map Torch's user-facing ``Shard(0)`` to ``BlockAtomic`` for MXFP8 groups |
| 634 | + and ``Flat`` for ordinary floating-point groups. |
635 | 635 | """ |
636 | | - if dtype not in (torch.float32, torch.bfloat16, torch.float16): |
637 | | - raise NotImplementedError(f"Unsupported dtype: {dtype}.") |
| 636 | + if group_dtype not in (torch.uint8, torch.float32, torch.bfloat16, torch.float16): |
| 637 | + raise NotImplementedError(f"Unsupported group dtype: {group_dtype}.") |
638 | 638 | for placement in placements: |
639 | 639 | if type(placement) is Shard and placement.dim != 0: |
640 | 640 | raise NotImplementedError( |
641 | 641 | "MFSDP currently supports only dim-0 Shard placements, " f"got {placement!r}." |
642 | 642 | ) |
643 | | - return tuple(Flat() if type(placement) is Shard else placement for placement in placements) |
| 643 | + placement_type = BlockAtomic(32) if group_dtype == torch.uint8 else Flat() |
| 644 | + return tuple( |
| 645 | + placement_type if type(placement) is Shard else placement for placement in placements |
| 646 | + ) |
0 commit comments