Skip to content

Commit d0f38f8

Browse files
committed
Clarify DBuffer allgather placement handling
1 parent 36bb55e commit d0f38f8

1 file changed

Lines changed: 7 additions & 6 deletions

File tree

  • megatron/core/distributed/fsdp/src/megatron_fsdp/experimental

‎megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/dbuffer.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -423,20 +423,21 @@ def redistribute(
423423
)
424424

425425
def allgather(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer":
426-
"""All-gather a Flat axis into Replicate placement."""
426+
"""All-gather a sharded axis into Replicate placement."""
427427
_validate_mesh_axis(self.mesh, mesh_axis)
428-
axis = mesh_axis
429-
if not isinstance(self.placements[axis], Flat):
430-
raise ValueError(f"allgather() requires Flat placement on axis {mesh_axis!r}.")
428+
if not isinstance(self.placements[mesh_axis], Flat):
429+
raise ValueError(
430+
f"allgather() currently requires Flat placement on axis {mesh_axis!r}."
431+
)
431432

432433
placements = list(self.placements)
433-
placements[axis] = Replicate()
434+
placements[mesh_axis] = Replicate()
434435
validate_placements(placements)
435436
out = self._create_or_validate_out(placements, out)
436437
dist.all_gather_into_tensor(
437438
output_tensor=out.local_buffer,
438439
input_tensor=self.local_buffer,
439-
group=self.mesh.get_group(axis),
440+
group=self.mesh.get_group(mesh_axis),
440441
)
441442
return out
442443

0 commit comments

Comments
 (0)