Skip to content

Commit 81a2379

Browse files
Qualcomm AI Engine Direct - Enhance Decompose im2col to support case … (pytorch#22634)
### Summary To solve the issue mentioned in pytorch#19690. The issue requires support for case kernel != stride and padding!=0. ``` dequantize_per_tensor_892: "f32[1, 1, 312, 552][172224, 172224, 552, 1]cuda:0" = torch.ops.quantized_decomposed.dequantize_per_tensor.default(quantize_per_tensor_590, 0.17020832002162933, 0, 0, 255, torch.uint8); quantize_per_tensor_590 = None im2col: "f32[1, 9, 172224][1550016, 172224, 1]cuda:0" = torch.ops.aten.im2col.default(dequantize_per_tensor_892, [3, 3], [1, 1], [1, 1], [1, 1]); dequantize_per_tensor_892 = None ``` - Rewrite the pass to support case kernel != stride and padding!=0. - Replace pixel_unshuffle to qnn custom op since torch pixel_unshuffle op doesn't support different block_size. - Remove assertion error so op not supported by QNN fallback to cpu. ### Test plan ``` python backends/qualcomm/tests/test_qnn_delegate.py -k "test_qnn_backend_unfold" --device ef5e4029 --host localhost --soc_model SM8850 --build_folder build-android --executorch_root . --artifact test_qnn_delegate_artifact python backends/qualcomm/tests/test_passes.py -k "test_decompose_im2col" pytest backends/qualcomm/tests/rework/htp/op/v68/test.py --build_folder build-android --soc_model SM8850 --device ef5e4029 -k test_unfold pytest backends/qualcomm/tests/rework/passes/test.py -k "decompose_col_im" ``` cc @cbilgin @psiddh
1 parent b6aed42 commit 81a2379

11 files changed

Lines changed: 562 additions & 144 deletions

File tree

‎backends/qualcomm/_passes/decompose_col_im.py‎

Lines changed: 191 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -3,27 +3,117 @@
33
#
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
6+
import logging
7+
68
import torch
79
from executorch.exir.dialects._ops import ops as exir_ops
810
from executorch.exir.pass_base import ExportPass, PassResult
911
from executorch.exir.passes import dead_code_elimination_pass
1012

11-
from .utils import copy_meta
13+
from .utils import copy_meta, create_const_node
1214

1315

1416
class DecomposeColIm(ExportPass):
1517
"""
16-
Decompose im2col(unfold) to pixel_unshuffle + view_copy
18+
Decompose im2col(unfold) to pad + index_select + space_to_depth + view_copy
1719
Decompose col2im(fold) to view_copy + pixel_shuffle
1820
"""
1921

22+
# When stride != kernel_size, the index_select gather in
23+
# _extend_for_space_to_depth duplicates elements. Skip decomposition if the
24+
# expansion_factor is too large to avoid unbounded memory consumption;
25+
# raise the value if a real case would benefit from more overlap.
26+
_MAX_GATHER_EXPANSION_FACTOR = 16
27+
2028
def __init__(self):
2129
super(DecomposeColIm, self).__init__()
2230
self.im2col_op = exir_ops.edge.aten.im2col.default
2331
self.col2im_op = exir_ops.edge.aten.col2im.default
2432
self.pixel_unshuffle_op = exir_ops.edge.aten.pixel_unshuffle.default
2533
self.pixel_shuffle_op = exir_ops.edge.aten.pixel_shuffle.default
2634
self.view_copy_op = exir_ops.edge.aten.view_copy.default
35+
self.index_select_op = exir_ops.edge.aten.index_select.default
36+
self.space_to_depth_op = exir_ops.edge.qnn_custom.space_to_depth.default
37+
self.pad_op = exir_ops.edge.aten.constant_pad_nd.default
38+
39+
def _emit_space_to_depth(
40+
self,
41+
graph: torch.fx.Graph,
42+
meta,
43+
input_node: torch.fx.Node,
44+
kernel_size: list[int],
45+
):
46+
kernel_height, kernel_width = kernel_size
47+
input_val = input_node.meta["val"]
48+
val = self.space_to_depth_op(input_val, kernel_height, kernel_width)
49+
node = graph.create_node(
50+
"call_function",
51+
self.space_to_depth_op,
52+
(input_node, kernel_height, kernel_width),
53+
)
54+
node.meta = copy_meta(meta)
55+
node.meta["val"] = val
56+
return node
57+
58+
def _extend_for_space_to_depth(
59+
self,
60+
graph_module: torch.fx.GraphModule,
61+
node: torch.fx.Node,
62+
input_node: torch.fx.Node,
63+
kernel_size,
64+
stride,
65+
padding,
66+
):
67+
graph = graph_module.graph
68+
if any(p != 0 for p in padding):
69+
# aten.im2col zero-pads symmetrically on both sides
70+
# of each spatial dim before extracting windows;
71+
# constant_pad_nd takes amounts last-dim-first:
72+
# [width_left, width_right, height_top, height_bottom].
73+
pad_amount = [padding[1], padding[1], padding[0], padding[0]]
74+
pad_val = self.pad_op(input_node.meta["val"], pad_amount, 0.0)
75+
pad_node = graph.create_node(
76+
"call_function", self.pad_op, (input_node, pad_amount, 0.0)
77+
)
78+
pad_node.meta = copy_meta(node.meta)
79+
pad_node.meta["val"] = pad_val
80+
input_node = pad_node
81+
# im2col with stride != kernel_size: gather each spatial dim (one 1-D
82+
# index_select per dim) so the result tiles exactly into
83+
# output_block_height*kernel_height by output_block_width*kernel_width -- i.e.
84+
# what space_to_depth expects when stride == kernel_size, matching
85+
# torch.nn.functional.unfold's output layout:
86+
# x.index_select(2, height_idx).index_select(3, width_idx)
87+
88+
for kernel_value, stride_value, dim in zip(kernel_size, stride, [2, 3]):
89+
input_val = input_node.meta["val"]
90+
input_dim_value = input_val.shape[dim]
91+
if kernel_value != stride_value:
92+
num_output_block = (input_dim_value - kernel_value) // stride_value + 1
93+
expand_idx = [
94+
j * stride_value + i
95+
for j in range(num_output_block)
96+
for i in range(kernel_value)
97+
]
98+
idx_node = create_const_node(
99+
graph,
100+
graph_module,
101+
f"{node.name}_dim_{dim}_expand_idx",
102+
expand_idx,
103+
input_node,
104+
const_dtype=torch.int32,
105+
static_shapes=True, # use static or it become symbolic shape
106+
)
107+
gather_node = graph.create_node(
108+
"call_function", self.index_select_op, (input_node, dim, idx_node)
109+
)
110+
gather_node.meta = copy_meta(node.meta)
111+
gather_node.meta["val"] = input_val.index_select(
112+
dim, idx_node.meta["val"]
113+
)
114+
input_node = gather_node
115+
116+
return input_node
27117

28118
def _decompose_im2col(self, graph_module: torch.fx.GraphModule):
29119
for node in graph_module.graph.nodes:
@@ -33,51 +123,78 @@ def _decompose_im2col(self, graph_module: torch.fx.GraphModule):
33123
dilation = node.args[2]
34124
padding = node.args[3]
35125
stride = node.args[4]
36-
batch_size = node.meta["val"].shape[0]
37-
assert (
38-
stride == kernel_size
39-
), "im2col can only be converted when stride == kernel_size"
40-
assert (
41-
input_node.meta["val"].dim() == 4
42-
), "im2col can only be converted when input dims == 4"
43-
assert (
44-
kernel_size[0] == kernel_size[1]
45-
), "im2col can only be converted when kernel height == width"
46-
assert all(
47-
d == 1 for d in dilation
48-
), "col2im can only be converted when dilation equals to (1, 1)"
49-
assert all(
50-
p == 0 for p in padding
51-
), "col2im can only be converted when padding equals to (0, 0)"
126+
if input_node.meta["val"].dim() != 4:
127+
logging.warning(
128+
f"{node}: im2col input must be 4-D to be decomposed, got "
129+
f"dim={input_node.meta['val'].dim()}. Skipping decomposition."
130+
)
131+
continue
132+
if any(d != 1 for d in dilation):
133+
logging.warning(
134+
f"{node}: im2col can only be decomposed when dilation == "
135+
f"(1, 1), got dilation={dilation}. Skipping decomposition."
136+
)
137+
continue
138+
139+
# Given a array length L with kernel K and stride S, the
140+
# expanded output is |1...K|1+S...K+S|...|1+(N-1)*S...K+(N-1)*S|
141+
# the expanded output length is K*N, where N = (L-K)/S+1,
142+
# expansion_factor = K*N/L = K*(L-K+S)/(L*S)
143+
# e.g., L=9, K=5, S=2, then expanded output is
144+
# |1...5|3...7|5...9|, expansion_factor = 5*(9-5+2)/(9*2) = 30/18
145+
# Assume L is largely greater than K and S, simplify
146+
# expansion_factor to K/S
147+
expansion_factor = (
148+
kernel_size[0] / stride[0] * kernel_size[1] / stride[1]
149+
)
150+
if expansion_factor > self._MAX_GATHER_EXPANSION_FACTOR:
151+
logging.warning(
152+
f"{node}: im2col decomposition would expand the gather "
153+
f"by {expansion_factor}x (kernel_size={kernel_size}, "
154+
f"stride={stride}), exceeding the "
155+
f"{self._MAX_GATHER_EXPANSION_FACTOR}x limit imposed to "
156+
"avoid excessive memory consumption. Skipping "
157+
"decomposition -- raise "
158+
"DecomposeColIm._MAX_GATHER_EXPANSION_FACTOR if a real "
159+
"case would benefit from loosening this threshold."
160+
)
161+
continue
162+
graph = graph_module.graph
52163
users = list(node.users.keys())
53-
with graph_module.graph.inserting_after(input_node):
54-
pixel_unshuffle_node = graph_module.graph.create_node(
55-
"call_function",
56-
self.pixel_unshuffle_op,
57-
(input_node, kernel_size[0]),
164+
x = input_node
165+
with graph.inserting_before(node):
166+
# given input [1,1,4,4] with kernel=[2,2] and stride=[1,1]
167+
# e.g.,input
168+
# input = [[ 0, 1, 2, 3],
169+
# [ 4, 5, 6, 7],
170+
# [ 8, 9, 10, 11],
171+
# [12, 13, 14, 15]]
172+
# since kernel is not equal to stride
173+
# it will be extend to a input shape with kernel=stride=[2,2]
174+
# e.g.,
175+
# input = [[ 0, 1, 1, 2, 2, 3],
176+
# [ 4, 5, 5, 6, 6, 7],
177+
# [ 4, 5, 5, 6, 6, 7],
178+
# [ 8, 9, 9, 10, 10, 11],
179+
# [ 8, 9, 9, 10, 10, 11],
180+
# [12, 13, 13, 14, 14, 15]]
181+
x = self._extend_for_space_to_depth(
182+
graph_module, node, x, kernel_size, stride, padding
58183
)
59-
pixel_unshuffle_node.meta = copy_meta(node.meta)
60-
orig_height = input_node.meta["val"].shape[2]
61-
orig_width = input_node.meta["val"].shape[3]
62184

63-
pixel_unshuffle_node.meta["val"] = pixel_unshuffle_node.meta[
64-
"val"
65-
].reshape(
66-
batch_size,
67-
-1,
68-
orig_height // kernel_size[0],
69-
orig_width // kernel_size[1],
185+
s2d_node = self._emit_space_to_depth(
186+
graph, node.meta, x, kernel_size
70187
)
71188

72-
with graph_module.graph.inserting_after(pixel_unshuffle_node):
73-
view_copy_node = graph_module.graph.create_node(
74-
"call_function",
75-
self.view_copy_op,
76-
(pixel_unshuffle_node, tuple(node.meta["val"].shape)),
77-
)
78-
view_copy_node.meta = copy_meta(node.meta)
79-
for user in users:
80-
user.replace_input_with(node, view_copy_node)
189+
final_view_node = graph.create_node(
190+
"call_function",
191+
self.view_copy_op,
192+
(s2d_node, tuple(node.meta["val"].shape)),
193+
)
194+
final_view_node.meta = copy_meta(node.meta)
195+
196+
for user in users:
197+
user.replace_input_with(node, final_view_node)
81198

82199
def _decompose_col2im(self, graph_module: torch.fx.GraphModule):
83200
for node in graph_module.graph.nodes:
@@ -89,21 +206,38 @@ def _decompose_col2im(self, graph_module: torch.fx.GraphModule):
89206
padding = node.args[4]
90207
stride = node.args[5]
91208
batch_size = node.meta["val"].shape[0]
92-
assert (
93-
stride == kernel_size
94-
), "col2im can only be converted when stride == kernel_size"
95-
assert (
96-
node.meta["val"].dim() == 4
97-
), "col2im can only be converted when output dims == 4"
98-
assert (
99-
kernel_size[0] == kernel_size[1]
100-
), "col2im can only be converted when kernel height == width"
101-
assert all(
102-
d == 1 for d in dilation
103-
), "col2im can only be converted when dilation equals to (1, 1)"
104-
assert all(
105-
p == 0 for p in padding
106-
), "col2im can only be converted when padding equals to (0, 0)"
209+
if stride != kernel_size:
210+
logging.warning(
211+
f"{node}: col2im can only be decomposed when stride == "
212+
f"kernel_size, got stride={stride}, kernel_size="
213+
f"{kernel_size}. Skipping decomposition."
214+
)
215+
continue
216+
if node.meta["val"].dim() != 4:
217+
logging.warning(
218+
f"{node}: col2im output must be 4-D to be decomposed, got "
219+
f"dim={node.meta['val'].dim()}. Skipping decomposition."
220+
)
221+
continue
222+
if kernel_size[0] != kernel_size[1]:
223+
logging.warning(
224+
f"{node}: col2im can only be decomposed when kernel "
225+
f"height == width, got kernel_size={kernel_size}. "
226+
f"Skipping decomposition."
227+
)
228+
continue
229+
if any(d != 1 for d in dilation):
230+
logging.warning(
231+
f"{node}: col2im can only be decomposed when dilation == "
232+
f"(1, 1), got dilation={dilation}. Skipping decomposition."
233+
)
234+
continue
235+
if any(p != 0 for p in padding):
236+
logging.warning(
237+
f"{node}: col2im can only be decomposed when padding == "
238+
f"(0, 0), got padding={padding}. Skipping decomposition."
239+
)
240+
continue
107241

108242
users = list(node.users.keys())
109243
with graph_module.graph.inserting_after(input_node):

‎backends/qualcomm/_passes/layout_transform.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ class LayoutTransform(ExportPass):
5656
exir_ops.edge.aten.native_group_norm.default,
5757
exir_ops.edge.aten.pixel_shuffle.default,
5858
exir_ops.edge.aten.pixel_unshuffle.default,
59+
exir_ops.edge.qnn_custom.space_to_depth.default,
5960
exir_ops.edge.aten.upsample_bicubic2d.default,
6061
exir_ops.edge.aten.upsample_bicubic2d.vec,
6162
exir_ops.edge.aten.upsample_bilinear2d.default,

‎backends/qualcomm/_passes/utils.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,19 +379,25 @@ def create_const_node(
379379
attr_name: str,
380380
value,
381381
source_node: torch.fx.Node,
382+
const_dtype: torch.dtype = None,
383+
static_shapes: bool = False,
382384
) -> torch.fx.Node:
383385
"""
384386
Register a scalar constant as a named buffer on the graph module and return a get_attr node referencing it.
385387
Used in edge dialect op decomposition passes where raw scalar arguments are not accepted by QNN op builders which need the inputs to be graph nodes.
386388
"""
387389
dtype = source_node.meta["val"].dtype
390+
if const_dtype is not None:
391+
dtype = const_dtype
388392
tensor = torch.tensor(value, dtype=dtype)
389393
graph_module.register_buffer(attr_name, tensor)
390394

391395
fake_mode = source_node.meta["val"].fake_mode
392396
with graph.inserting_before(next(iter(graph.nodes))):
393397
const_node = graph.get_attr(attr_name)
394-
const_node.meta["val"] = fake_mode.from_tensor(tensor)
398+
const_node.meta["val"] = fake_mode.from_tensor(
399+
tensor, static_shapes=static_shapes
400+
)
395401
return const_node
396402

397403

‎backends/qualcomm/builders/custom_ops.py‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,19 @@ def _hadamard_matrix(dim: int, device, dtype) -> torch.Tensor:
1616
return h
1717

1818

19-
if not hasattr(torch.ops, "qnn_custom") or not hasattr(
20-
torch.ops.qnn_custom, "hadamard_transform"
19+
if (
20+
not hasattr(torch.ops, "qnn_custom")
21+
or not hasattr(torch.ops.qnn_custom, "hadamard_transform")
22+
or not hasattr(torch.ops.qnn_custom, "space_to_depth")
2123
):
22-
hadamard_op_lib = Library("qnn_custom", "DEF")
23-
hadamard_op_lib.define("hadamard_transform(Tensor input, float scale) -> Tensor")
24+
qnn_custom_lib = Library("qnn_custom", "DEF")
25+
qnn_custom_lib.define("hadamard_transform(Tensor input, float scale) -> Tensor")
26+
qnn_custom_lib.define(
27+
"space_to_depth(Tensor input, int block_h, int block_w) -> Tensor"
28+
)
2429

25-
@impl(hadamard_op_lib, "hadamard_transform", "CompositeExplicitAutograd")
30+
# Qnn HadamardTransform
31+
@impl(qnn_custom_lib, "hadamard_transform", "CompositeExplicitAutograd")
2632
def hadamard_transform_impl(input: torch.Tensor, scale: float) -> torch.Tensor:
2733
# Normalized Walsh-Hadamard transform along the last dim, times scale.
2834
# Matches a linear/matmul whose weight is scipy.linalg.hadamard(dim) * s,
@@ -37,5 +43,27 @@ def hadamard_transform_fake(input: torch.Tensor, scale: float) -> torch.Tensor:
3743
# Hadamard weight is square, so the transform preserves shape.
3844
return torch.empty_like(input)
3945

46+
# Qnn SpaceToDepth
47+
@impl(qnn_custom_lib, "space_to_depth", "CompositeExplicitAutograd")
48+
def space_to_depth_impl(
49+
input: torch.Tensor, block_h: int, block_w: int
50+
) -> torch.Tensor:
51+
# Generalizes torch.nn.functional.pixel_unshuffle to independent block
52+
# sizes along height/width, matching QNN SpaceToDepth's CRD mode: output
53+
# channel c*block_h*block_w + i*block_w + j holds input channel c's
54+
# (i, j)'th block element.
55+
n, c, h, w = input.shape
56+
out_h, out_w = h // block_h, w // block_w
57+
x = input.view(n, c, out_h, block_h, out_w, block_w)
58+
x = x.permute(0, 1, 3, 5, 2, 4)
59+
return x.reshape(n, c * block_h * block_w, out_h, out_w)
60+
61+
@register_fake("qnn_custom::space_to_depth")
62+
def space_to_depth_fake(
63+
input: torch.Tensor, block_h: int, block_w: int
64+
) -> torch.Tensor:
65+
n, c, h, w = input.shape
66+
return input.new_empty(n, c * block_h * block_w, h // block_h, w // block_w)
67+
4068
else:
41-
hadamard_op_lib = Library("qnn_custom", "FRAGMENT")
69+
qnn_custom_lib = Library("qnn_custom", "FRAGMENT")

0 commit comments

Comments
 (0)