Skip to content

Commit e5311bf

Browse files
Arm backend: Add partial 3d support for gather (pytorch#16822)
Add support for 3D gathers when input_size[-1] == index_size[-1]. In this case we can permute and reshape input [N,K,C] and index [N,W,C] to [N*C,K,1] and [N*C,W], apply a tosa.GATHER, and then reshape and permute back to [N,W,C]. cc @freddan80 @per @zingo @digantdesai Signed-off-by: Oscar Andersson <oscar.andersson@arm.com>
1 parent 7d7e89a commit e5311bf

3 files changed

Lines changed: 168 additions & 55 deletions

File tree

‎backends/arm/_passes/canonicalize_gather_pass.py‎

Lines changed: 122 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,38 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
7-
import logging
86
from typing import Set, Type
97

108
import torch
119
from executorch.backends.arm._passes import ArmPass
1210
from executorch.exir.dialects._ops import ops as exir_ops
1311
from executorch.exir.pass_base import ExportPass
1412

15-
logger = logging.getLogger(__name__)
16-
1713

1814
class CanonicalizeGatherPass(ArmPass):
1915
"""
2016
Canonicalize gather so it can be lowered to TOSA.GATHER via the backend dialect.
2117
2218
This pass is intended to run only for nodes already gated by GatherSupported.
2319
24-
Behavior:
25-
- Reshape x from [N,K] to [N,K,1] so values matches TOSA gather's [N,K,C].
26-
- Keep indices as [N,W]
27-
- Lower using tosa.GATHER.default, producing [N,W,1].
28-
- Reshape output to [N,W].
29-
- Only insert bool<->int8 casts when x is bool:
20+
Generic behavior:
21+
- Only insert bool<->int8 casts when x is bool:
3022
* If x is bool: gather runs on int8 and output is cast back to bool.
3123
* If x is not bool: gather runs on original dtype and output keeps dtype.
24+
25+
2D behavior:
26+
- Reshape x from [N,K] to [N,K,1] so values matches TOSA gather's [N,K,C].
27+
- Reshape indices to [N,W].
28+
- Lower using tosa.GATHER.default, producing [N,W,1].
29+
- Reshape output to [N,W].
30+
31+
3D behavior:
32+
- Permute and reshape x from [N,K,C] to [N*C,K,1].
33+
- Permute and reshape indices from [N,W,C] to [N*C,W].
34+
- Lower using tosa.GATHER.default, producing [N*C,W,1].
35+
- Reshape and permute output to [N,W,C].
36+
- Note that this decomposition requires the channel size of the indices to be the same as the channel
37+
size of the input, which is not guaranteed in Pytorch. We reject nodes not fulfilling this when partitioning.
3238
"""
3339

3440
_passes_required_after: Set[Type[ExportPass]] = set()
@@ -45,15 +51,17 @@ def call_operator(self, op, args, kwargs, meta):
4551
# GatherSupported should have gated this already; treat violations as errors.
4652
x_shape = x.data.shape
4753
index_shape = index.data.shape
54+
dim = dim % len(x_shape)
4855
if not (
49-
dim in (1, -1)
50-
and len(x_shape) == 2
51-
and len(index_shape) == 2
56+
dim in (1,)
57+
and len(x_shape) in (2, 3)
58+
and len(index_shape) in (2, 3)
5259
and index_shape[0] == x_shape[0]
5360
):
5461
raise RuntimeError(
55-
f"[{op}] Unexpected gather pattern; expected "
56-
f"x:[N,K], index:[N,W], dim in {{1,-1}}, matching N. "
62+
f"[{op}] Unexpected gather pattern; expected one of: "
63+
f"x:[N,K], index:[N,W], dim in {{1,}}, matching N, "
64+
f"x:[N,K,C], index:[N,W,C], dim in {{1,}}, matching N. "
5765
f"Got dim={dim}, x.shape={x_shape}, index.shape={index_shape}."
5866
)
5967

@@ -62,6 +70,7 @@ def call_operator(self, op, args, kwargs, meta):
6270

6371
view_op = exir_ops.edge.aten.view_copy.default
6472
to_copy_op = exir_ops.edge.dim_order_ops._to_dim_order_copy.default
73+
permute_copy_op = exir_ops.edge.aten.permute_copy.default
6574

6675
# Use backend dialect gather:
6776
# values: [N,K,C]
@@ -83,40 +92,111 @@ def call_operator(self, op, args, kwargs, meta):
8392
)
8493

8594
# [N,K] -> [N,K,1]
86-
values_3d = super().call_operator(
87-
view_op,
88-
(values_in, [N, K, 1]),
89-
{},
90-
meta,
91-
updated=True,
92-
)
93-
94-
# indices stays [N,W]
95-
gathered_3d = super().call_operator(
96-
tosa_gather_op,
97-
(values_3d, index),
98-
{},
99-
meta,
100-
updated=True,
101-
)
102-
103-
# [N,W,1] -> [N,W]
104-
gathered_2d = super().call_operator(
105-
view_op,
106-
(gathered_3d, [N, W]),
107-
{},
108-
meta,
109-
updated=True,
110-
)
95+
if len(x_shape) < 3:
96+
values_3d = super().call_operator(
97+
view_op,
98+
(values_in, [N, K, 1]),
99+
{},
100+
meta,
101+
updated=True,
102+
)
103+
104+
# indices stays [N,W]
105+
gathered_3d = super().call_operator(
106+
tosa_gather_op,
107+
(values_3d, index),
108+
{},
109+
meta,
110+
updated=True,
111+
)
112+
else:
113+
# For 3D inputs we need to reshuffle both input and indices so that
114+
# the payload is accessed from the batch rather than the channels
115+
# since torch allows for accessing of different indices across channels
116+
# while TOSA does not.
117+
# 1. Permute input [N,K,C] to [N,C,K].
118+
# 2. Rehsape input [N,K,C] to [N*C,K,1].
119+
# 3. Permute indices [N,W,C] to [N,C,W].
120+
# 4. Reshape indices [N,C,W] to [N*C,W].
121+
# 5. Gather using tosa.GATHER with reshaped inputs and indices.
122+
# 6. Reshape output [N*C,W,1] to [N,C,W].
123+
# 7. Permute output [N,C,W] to [N,W,C].
124+
125+
# TODO if index input is a repeat across last dim with C repeats
126+
# we can remove the repeat and directly lower to tosa.GATHER.
127+
values_3d = values_in
128+
N, K, C = x_shape
129+
_, W, _ = index_shape
130+
values_3d_permuted = super().call_operator(
131+
permute_copy_op,
132+
(values_3d, [0, 2, 1]),
133+
{},
134+
meta,
135+
updated=True,
136+
) # [N,C,K]
137+
values_3d_reshaped = super().call_operator(
138+
view_op,
139+
(values_3d_permuted, [N * C, K, 1]),
140+
{},
141+
meta,
142+
updated=True,
143+
) # [N*C,K,1]
144+
indices_permuted = super().call_operator(
145+
permute_copy_op,
146+
(index, [0, 2, 1]),
147+
{},
148+
meta,
149+
updated=True,
150+
) # [N,L,W]
151+
indices_reshaped = super().call_operator(
152+
view_op,
153+
(indices_permuted, [N * C, W]),
154+
{},
155+
meta,
156+
updated=True,
157+
) # [N*L,W]
158+
gathered = super().call_operator(
159+
tosa_gather_op,
160+
(values_3d_reshaped, indices_reshaped),
161+
{},
162+
meta,
163+
updated=True,
164+
) # [N*L,W,1]
165+
gathered_reshaped = super().call_operator(
166+
view_op,
167+
(gathered, [N, C, W]),
168+
{},
169+
meta,
170+
updated=True,
171+
) # [N,L,W]
172+
gathered_3d = super().call_operator(
173+
permute_copy_op,
174+
(gathered_reshaped, [0, 2, 1]),
175+
{},
176+
meta,
177+
updated=True,
178+
) # [N,W,L]
179+
180+
if len(x_shape) == 2:
181+
# [N,W,1] -> [N,W]
182+
gathered = super().call_operator(
183+
view_op,
184+
(gathered_3d, [N, W]),
185+
{},
186+
meta,
187+
updated=True,
188+
)
189+
else:
190+
gathered = gathered_3d
111191

112192
# int8 -> bool (only if needed)
113193
if needs_bool_cast:
114194
return super().call_operator(
115195
to_copy_op,
116-
(gathered_2d,),
196+
(gathered,),
117197
{"dtype": torch.bool},
118198
meta,
119199
updated=True,
120200
)
121201

122-
return gathered_2d
202+
return gathered

‎backends/arm/operator_support/gather_support.py‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@
99
1010
- target: exir_ops.edge.aten.gather.default
1111
- args: exactly (x, dim, index) (i.e. len(node.args) == 3)
12-
- dim must be 1 or -1
13-
- x must be rank-2
14-
- index must be rank-2
12+
- dim must map to 1
13+
- x must be rank-2 or rank-3
14+
- index must be rank-2 or rank-3.
15+
- for rank-3 x.shape[-1] must match index.shape[-1]
1516
- index dtype must be int32
1617
- batch dim must match: x.shape[0] == index.shape[0]
1718
@@ -23,10 +24,14 @@
2324
when running under an INT profile.
2425
2526
Note:
26-
- CanonicalizeGatherPass reshapes values to [N, K, 1] and keeps indices as [N, W],
27-
then lowers via the TOSA gather dialect.
27+
- For 2D inputs CanonicalizeGatherPass reshapes values to [N, K, 1] and keeps
28+
indices as [N, W],then lowers via the TOSA gather dialect.
29+
- For 3D inputs CanonicalizeGatherPass permutes and reshapes values and indices
30+
to [N*C, K, 1] and [N*C, W] respectively, then lowers via the TOSA gather dialect.
2831
"""
2932

33+
from typing import cast
34+
3035
import torch
3136
import torch.fx as fx
3237

@@ -66,6 +71,8 @@ def is_node_tosa_supported(
6671

6772
x_shape = tuple(x_val.shape)
6873
index_shape = tuple(index_val.shape)
74+
dim = cast(int, dim)
75+
dim = dim % len(x_shape)
6976

7077
# ---- index dtype ----
7178
if index_val.dtype != torch.int32:
@@ -77,17 +84,24 @@ def is_node_tosa_supported(
7784
return False
7885

7986
# ---- dim + rank ----
80-
if not (
81-
(dim == 1 or dim == -1) and len(x_shape) == 2 and len(index_shape) == 2
82-
):
87+
if not ((dim == 1) and len(x_shape) in (2, 3) and len(index_shape) in (2, 3)):
8388
self.reporter.report_reject(
8489
node,
8590
f"{node.target}: unsupported dim/rank; got {dim=}, "
8691
f"x_rank={len(x_shape)}, index_rank={len(index_shape)}; "
87-
"supported: dim in {1, -1} with rank-2 x and rank-2 index.",
92+
"supported: dim in {1,} with rank-2/3 x and rank-2/3 index.",
8893
)
8994
return False
9095

96+
if len(index_shape) == 3:
97+
if x_shape[-1] != index_shape[-1]:
98+
self.reporter.report_reject(
99+
node,
100+
f"{node.target}: trailing dimension size mismatch "
101+
f"{x_shape[-1]=} vs {index_shape[-1]=}.",
102+
)
103+
return False
104+
91105
# ---- batch dim compatibility ----
92106
if x_shape[0] != index_shape[0]:
93107
self.reporter.report_reject(

‎backends/arm/test/ops/test_gather.py‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def forward(self, input_: torch.Tensor, dim_, index_: torch.Tensor):
3232

3333
# FP profile: only float inputs.
3434
test_data_fp: dict[str, input_params] = {
35-
"test_fp32": (
35+
"test_fp32_2d": (
3636
torch.tensor(
3737
[[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3], [3.1, 3.2, 3.3]],
3838
dtype=torch.float32,
@@ -43,13 +43,32 @@ def forward(self, input_: torch.Tensor, dim_, index_: torch.Tensor):
4343
dtype=torch.int64,
4444
), # Shape: [N=4, W=2]
4545
),
46+
"test_fp32_3d": (
47+
torch.tensor(
48+
[
49+
[[0.1, 0.2, 0.3], [1.1, 1.2, 1.3], [2.1, 2.2, 2.3]],
50+
[[3.1, 3.2, 3.1], [4.1, 4.2, 4.3], [5.1, 5.2, 5.3]],
51+
[[6.1, 6.2, 6.3], [7.1, 7.2, 7.3], [8.1, 8.2, 8.3]],
52+
],
53+
dtype=torch.float32,
54+
), # Shape: [N=3, K=3, C=3]
55+
1,
56+
torch.tensor(
57+
[
58+
[[0, 1, 2], [1, 2, 1], [2, 0, 1]],
59+
[[1, 1, 2], [2, 2, 1], [0, 0, 1]],
60+
[[2, 1, 2], [0, 2, 1], [1, 0, 1]],
61+
],
62+
dtype=torch.int64,
63+
), # Shape: [N=3, W=3, C=3]
64+
),
4665
}
4766

4867

4968
# INT profile: integer inputs + bool (bool is supported via casts in
5069
# CanonicalizeGatherPass: bool -> int8 -> bool).
5170
test_data_int: dict[str, input_params] = {
52-
"test_int32": (
71+
"test_int32_2d": (
5372
torch.tensor(
5473
[[1, 2, 3], [11, 12, 13], [21, 22, 23], [31, 32, 33]],
5574
dtype=torch.int32,
@@ -60,15 +79,15 @@ def forward(self, input_: torch.Tensor, dim_, index_: torch.Tensor):
6079
dtype=torch.int64,
6180
), # Shape: [N=4, W=2]
6281
),
63-
"test_int8": (
82+
"test_int8_2d": (
6483
torch.randint(5, size=(5, 10), dtype=torch.int8), # Shape: [N=5, K=10]
6584
1,
6685
torch.tensor(
6786
[[3, 2, 1], [2, 4, 0], [1, 0, 2], [2, 3, 1], [4, 1, 2]],
6887
dtype=torch.int64,
6988
), # Shape: [N=5, W=3]
7089
),
71-
"test_bool": (
90+
"test_bool_2d": (
7291
torch.tensor(
7392
[
7493
[True, False, True],

0 commit comments

Comments
 (0)