Skip to content

Commit d9649f7

Browse files
[MLX] Support dynamic GroupNorm and upsample (pytorch#23450)
## Summary Adds dynamic spatial-shape support to the MLX `native_group_norm` and `upsample_nearest2d` handlers. GroupNorm now computes symbolic spatial products at runtime. Dynamic integer nearest-neighbour scaling uses repeat operations, while static shapes retain the general gather implementation. Tests cover exporting at one spatial size and executing at another. ## Files - `backends/mlx/ops.py`: dynamic GroupNorm and nearest-neighbour lowering - `backends/mlx/test/test_ops.py`: dynamic spatial tests ## Testing ```bash python -m executorch.backends.mlx.test.run_all_tests \ group_norm_dynamic \ upsample_nearest2d_dynamic
1 parent 27d124f commit d9649f7

2 files changed

Lines changed: 75 additions & 16 deletions

File tree

‎backends/mlx/ops.py‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2819,7 +2819,7 @@ def _native_group_norm_handler(P: MLXProgramBuilder, n: Node) -> Slot:
28192819
require_kwargs(P.kwargs(n), set(), "aten.native_group_norm")
28202820
x, weight, bias, N, C, HxW, group, eps = args
28212821

2822-
for name, value in (("N", N), ("C", C), ("HxW", HxW), ("group", group)):
2822+
for name, value in (("N", N), ("C", C), ("group", group)):
28232823
if not isinstance(value, int):
28242824
raise ValueError(
28252825
f"aten.native_group_norm requires a static {name}, got {value!r}"
@@ -2828,12 +2828,12 @@ def _native_group_norm_handler(P: MLXProgramBuilder, n: Node) -> Slot:
28282828
x_meta = n.args[0].meta.get("val")
28292829
if x_meta is None:
28302830
raise ValueError("aten.native_group_norm requires input shape metadata")
2831-
if any(not isinstance(d, int) for d in x_meta.shape):
2832-
raise ValueError(
2833-
f"aten.native_group_norm requires a static input shape, "
2834-
f"got {tuple(x_meta.shape)}"
2835-
)
28362831
x_ndim = len(x_meta.shape)
2832+
orig_shape = emit_shape(P, n.args[0], x)
2833+
normalized_width = emit_product(
2834+
P,
2835+
[IntOrVid.from_literal(C // group), P.to_int_or_vid(HxW)],
2836+
)
28372837

28382838
# native_group_norm returns (output, mean, rstd) -- allocate all 3 slots
28392839
output_slots = P.make_or_get_slots(n)
@@ -2846,7 +2846,7 @@ def _native_group_norm_handler(P: MLXProgramBuilder, n: Node) -> Slot:
28462846
out=P.slot_to_tid(flat),
28472847
shape=[
28482848
IntOrVid.from_literal(N * group),
2849-
IntOrVid.from_literal((C // group) * HxW),
2849+
normalized_width,
28502850
],
28512851
)
28522852
)
@@ -2862,7 +2862,6 @@ def _native_group_norm_handler(P: MLXProgramBuilder, n: Node) -> Slot:
28622862
)
28632863
)
28642864

2865-
orig_shape = [IntOrVid.from_literal(int(d)) for d in x_meta.shape]
28662865
if weight is None and bias is None:
28672866
P.emit(
28682867
ReshapeNode(
@@ -2950,12 +2949,11 @@ def _nearest_source_indices(in_size: int, out_size: int, scale: Optional[float])
29502949
]
29512950
)
29522951
def _upsample_nearest2d_handler(P: MLXProgramBuilder, n: Node) -> Slot:
2953-
"""Nearest-neighbour 2D resampling as two gathers.
2952+
"""Nearest-neighbour 2D resampling as two repeats or gathers.
29542953
2955-
Each output position reads a source position that depends only on the static
2956-
input and output sizes, so both index vectors are constants and the op is
2957-
``take(take(x, idx_h, -2), idx_w, -1)``. Expressing it as a gather rather
2958-
than a repeat also covers non-integer scale factors and downsampling.
2954+
Integer scale factors use repeats, which also support dynamic spatial
2955+
dimensions. Other static shapes use gathers to cover non-integer scale
2956+
factors and downsampling.
29592957
"""
29602958
args = P.args(n)
29612959
kwargs = P.kwargs(n)
@@ -2989,10 +2987,35 @@ def _upsample_nearest2d_handler(P: MLXProgramBuilder, n: Node) -> Slot:
29892987

29902988
sizes = (x_meta.shape[-2], x_meta.shape[-1], out_meta.shape[-2], out_meta.shape[-1])
29912989
if any(not isinstance(d, int) for d in sizes):
2992-
raise ValueError(
2993-
f"aten.upsample_nearest2d requires static spatial sizes, got "
2994-
f"{tuple(x_meta.shape)} -> {tuple(out_meta.shape)}"
2990+
if not all(
2991+
isinstance(scale, (int, float)) and scale > 0 and float(scale).is_integer()
2992+
for scale in scales
2993+
):
2994+
raise ValueError(
2995+
f"aten.upsample_nearest2d requires positive integer scale factors "
2996+
f"for dynamic spatial sizes, got {scales}"
2997+
)
2998+
2999+
_, rows = P.make_tmp_slot()
3000+
P.emit(
3001+
RepeatNode(
3002+
x=P.slot_to_tid(x),
3003+
out=P.slot_to_tid(rows),
3004+
repeats=IntOrVid.from_literal(int(scales[0])),
3005+
axis=-2,
3006+
)
29953007
)
3008+
out = P.make_or_get_slot(n)
3009+
P.emit(
3010+
RepeatNode(
3011+
x=P.slot_to_tid(rows),
3012+
out=P.slot_to_tid(out),
3013+
repeats=IntOrVid.from_literal(int(scales[1])),
3014+
axis=-1,
3015+
)
3016+
)
3017+
return out
3018+
29963019
in_h, in_w, out_h, out_w = sizes
29973020

29983021
index_slots = []

‎backends/mlx/test/test_ops.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3364,13 +3364,15 @@ def __init__(
33643364
shape: Tuple[int, ...] = (2, 32, 8, 8),
33653365
eps: float = 1e-5,
33663366
affine: bool = True,
3367+
dynamic_spatial: bool = False,
33673368
suffix: str = "",
33683369
):
33693370
self.num_groups = num_groups
33703371
self.num_channels = num_channels
33713372
self.shape = shape
33723373
self.eps = eps
33733374
self.affine = affine
3375+
self.dynamic_spatial = dynamic_spatial
33743376
self.name = f"group_norm{suffix}"
33753377

33763378
@classmethod
@@ -3385,6 +3387,7 @@ def get_test_configs(cls) -> List["GroupNormTest"]:
33853387
# non-square spatial extent, and a 3D (N, C, L) input
33863388
cls(num_groups=4, num_channels=16, shape=(1, 16, 5, 7), suffix="_odd"),
33873389
cls(num_groups=4, num_channels=12, shape=(2, 12, 7), suffix="_3d"),
3390+
cls(shape=(1, 32, 8, 8), dynamic_spatial=True, suffix="_dynamic"),
33883391
]
33893392

33903393
def create_model(self) -> nn.Module:
@@ -3393,6 +3396,21 @@ def create_model(self) -> nn.Module:
33933396
def create_inputs(self) -> Tuple[torch.Tensor, ...]:
33943397
return (torch.randn(*self.shape),)
33953398

3399+
def create_test_inputs(self) -> Tuple[torch.Tensor, ...]:
3400+
if self.dynamic_spatial:
3401+
return (torch.randn(self.shape[0], self.shape[1], 12, 16),)
3402+
return self.create_inputs()
3403+
3404+
def get_dynamic_shapes(self) -> Optional[Dict]:
3405+
if self.dynamic_spatial:
3406+
return {
3407+
"x": {
3408+
2: Dim("height", min=4, max=32),
3409+
3: Dim("width", min=4, max=32),
3410+
}
3411+
}
3412+
return None
3413+
33963414

33973415
class UpsampleNearest2dModel(nn.Module):
33983416
"""Nearest-neighbour resize, by scale factor or by explicit output size."""
@@ -3425,17 +3443,20 @@ def __init__(
34253443
shape: Tuple[int, ...] = (1, 3, 4, 4),
34263444
scale_factor: Optional[Tuple[float, float]] = (2.0, 2.0),
34273445
size: Optional[Tuple[int, int]] = None,
3446+
dynamic_spatial: bool = False,
34283447
suffix: str = "",
34293448
):
34303449
self.shape = shape
34313450
self.scale_factor = scale_factor
34323451
self.size = size
3452+
self.dynamic_spatial = dynamic_spatial
34333453
self.name = f"upsample_nearest2d{suffix}"
34343454

34353455
@classmethod
34363456
def get_test_configs(cls) -> List["UpsampleNearest2dTest"]:
34373457
return [
34383458
cls(),
3459+
cls(dynamic_spatial=True, suffix="_dynamic"),
34393460
# different scale per axis
34403461
cls(shape=(2, 5, 3, 7), scale_factor=(3.0, 2.0), suffix="_anisotropic"),
34413462
# non-integer ratios, which a repeat-based lowering could not express
@@ -3461,6 +3482,21 @@ def create_model(self) -> nn.Module:
34613482
def create_inputs(self) -> Tuple[torch.Tensor, ...]:
34623483
return (torch.randn(*self.shape),)
34633484

3485+
def create_test_inputs(self) -> Tuple[torch.Tensor, ...]:
3486+
if self.dynamic_spatial:
3487+
return (torch.randn(self.shape[0], self.shape[1], 6, 10),)
3488+
return self.create_inputs()
3489+
3490+
def get_dynamic_shapes(self) -> Optional[Dict]:
3491+
if self.dynamic_spatial:
3492+
return {
3493+
"x": {
3494+
2: Dim("height", min=2, max=16),
3495+
3: Dim("width", min=2, max=16),
3496+
}
3497+
}
3498+
return None
3499+
34643500

34653501
class Conv1dModel(nn.Module):
34663502
"""Simple model using Conv1d."""

0 commit comments

Comments
 (0)