Skip to content

Commit d386a04

Browse files
authored
Arm backend: Fix TransposeConv checks for U55 (pytorch#18414)
Make the TransposeConv2D checks for U55 compliant with Vela 4.5.0. Fixes the bug reported in pytorch#17668
1 parent 5e77594 commit d386a04

2 files changed

Lines changed: 114 additions & 19 deletions

File tree

‎backends/arm/operator_support/convolution_support.py‎

Lines changed: 76 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,82 @@ def _check_transposed_support(
124124
return False
125125
return True
126126

127+
def _check_transposed_conv_u55(self, node: fx.Node) -> bool:
128+
"""Implement the condition checks specified in
129+
https://gitlab.arm.com/artificial-intelligence/ethos-u/ethos-u-vela/-/blob/main/SUPPORTED_OPS.md?ref_type=heads#ethos-u55-and-ethos-u65-tosa-transpose_conv2d-constraints
130+
for the TransposeConv2D for Ethos-U55.
131+
"""
132+
input_fake_tensor = get_first_fake_tensor(cast(fx.Node, node.args[0]))
133+
if input_fake_tensor.dim() != 4:
134+
self.reporter.report_reject(
135+
node,
136+
f"TransposeConv2d requires 4D input, got rank {input_fake_tensor.dim()}.",
137+
)
138+
return False
139+
# For TransposeConv2D, the input tensor can only be 4D, hence we don't expect 3D input tensor.
140+
# In case of a 3D input tensor, the user should first unsqueeze the tensor and then pass it
141+
# to the TransposeConv2D, otherwise PyTorch throws an error.
142+
if input_fake_tensor.dim_order() == (0, 1, 2, 3):
143+
input_C = input_fake_tensor.shape[1]
144+
input_H = input_fake_tensor.shape[2]
145+
input_W = input_fake_tensor.shape[3]
146+
elif input_fake_tensor.dim_order() == (0, 2, 3, 1):
147+
input_C = input_fake_tensor.shape[3]
148+
input_H = input_fake_tensor.shape[1]
149+
input_W = input_fake_tensor.shape[2]
150+
else:
151+
self.reporter.report_reject(
152+
node,
153+
f"Unsupported dim order. Support dim orders (0,1,2,3) or (0,2,3,1) and got dim_order {input_fake_tensor.dim_order()}",
154+
)
155+
return False
156+
kernel = cast(fx.Node, node.args[1]).meta["val"].shape
157+
kernel_h = kernel[2]
158+
kernel_w = kernel[3] if len(kernel) > 3 else 1
159+
if (
160+
input_C > 65536
161+
or input_C < 1
162+
or input_H > 65536
163+
or input_H < 1
164+
or input_W > 65536
165+
or input_W < 1
166+
):
167+
self.reporter.report_reject(
168+
node,
169+
f"HWC must be in the range [1;65536] but got {input_C} {input_H} {input_W}",
170+
)
171+
return False
172+
173+
if (
174+
kernel_h * kernel_w > 4096
175+
or kernel_h * kernel_w < 1
176+
or kernel_h > 64
177+
or kernel_h < 1
178+
):
179+
self.reporter.report_reject(
180+
node,
181+
f"Kernel Height * Kernel Width must be in the range [1;4096] but got {kernel_h * kernel_w}",
182+
)
183+
return False
184+
strides = expand_around_channel(cast(list[int], node.args[3]), 2)
185+
stride = (strides[0], strides[1])
186+
ok = False
187+
if stride in ((1, 1), (2, 2)):
188+
ok = True
189+
elif stride == (1, 2):
190+
ok = input_H == 1 and kernel_h == 1
191+
elif stride == (2, 1):
192+
ok = input_W == 1 and kernel_w == 1
193+
if not ok:
194+
self.reporter.report_reject(
195+
node,
196+
f"Unsupported stride of {stride} for Ethos-U55. You can use stride of (1,1) or (2,2), stride (1,2) for IFM height kernel height of 1 and stride(2,1) for IFM width and kernel width of 1",
197+
)
198+
return False
199+
return True
200+
127201
def _is_node_supported_u55(self, node: fx.Node) -> bool:
128-
"""Enforce Ethos-U55-specific constraints (Vela 4.2.0).
202+
"""Enforce Ethos-U55-specific constraints (Vela 5.0.0).
129203
130204
Check channel dimensions, kernel sizes, and stride/pad/dilation
131205
combinations permitted on U55.
@@ -139,23 +213,7 @@ def _is_node_supported_u55(self, node: fx.Node) -> bool:
139213
"""
140214
transposed = cast(bool, node.args[6])
141215
if transposed:
142-
kernel = cast(fx.Node, node.args[1]).meta["val"].shape
143-
kernel_h = kernel[2]
144-
kernel_w = kernel[3] if len(kernel) > 3 else 1
145-
if kernel_h != kernel_w:
146-
self.reporter.report_reject(
147-
node,
148-
f"Transpose convolution on U55 requires square kernels, got ({kernel_w}, {kernel_h}).",
149-
)
150-
return False
151-
152-
strides = expand_around_channel(cast(list[int], node.args[3]), 2)
153-
if strides[0] != strides[1]:
154-
self.reporter.report_reject(
155-
node,
156-
f"Transpose convolution on U55 requires equal strides, got {strides}.",
157-
)
158-
return False
216+
return self._check_transposed_conv_u55(node)
159217

160218
shape_in = cast(torch.Tensor, node.all_input_nodes[0].meta["val"]).shape
161219
shape_out = node.meta["val"].shape

‎backends/arm/test/ops/test_transpose_conv2d.py‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,39 @@ def _get_per_channel_observers(module: torch.nn.Module):
177177
)
178178
}
179179

180+
u55_TransposeConv_limitations = {
181+
"stride_3": lambda: (
182+
TransposeConv2d(
183+
in_channels=3,
184+
out_channels=1,
185+
kernel_size=(5, 5),
186+
stride=(3, 3),
187+
padding=(2, 2),
188+
),
189+
True,
190+
),
191+
"stride_1_2": lambda: (
192+
TransposeConv2d(
193+
in_channels=3,
194+
out_channels=1,
195+
kernel_size=(5, 5),
196+
stride=(1, 2),
197+
padding=(2, 2),
198+
),
199+
True,
200+
),
201+
"stride_2_1": lambda: (
202+
TransposeConv2d(
203+
in_channels=3,
204+
out_channels=1,
205+
kernel_size=(5, 5),
206+
stride=(2, 1),
207+
padding=(2, 2),
208+
),
209+
True,
210+
),
211+
}
212+
180213
reject_suite = {
181214
k: v
182215
for k, v in test_data_INT.items()
@@ -568,7 +601,11 @@ def test_conv_transpose2d_u55_INT(test_data):
568601
}
569602

570603

571-
@common.parametrize("test_data", reject_suite, xfails=_u55_grouped_not_delegated_xfails)
604+
@common.parametrize(
605+
"test_data",
606+
reject_suite | u55_TransposeConv_limitations,
607+
xfails=_u55_grouped_not_delegated_xfails,
608+
)
572609
def test_conv_transpose2d_u55_INT_not_delegated(test_data):
573610
model, per_channel_quantization = test_data()
574611
OpNotSupportedPipeline(

0 commit comments

Comments
 (0)