33# This source code is licensed under the BSD-style license found in the
44# LICENSE file in the root directory of this source tree.
55
6-
7- import logging
86from typing import Set , Type
97
108import torch
119from executorch .backends .arm ._passes import ArmPass
1210from executorch .exir .dialects ._ops import ops as exir_ops
1311from executorch .exir .pass_base import ExportPass
1412
15- logger = logging .getLogger (__name__ )
16-
1713
1814class CanonicalizeGatherPass (ArmPass ):
1915 """
2016 Canonicalize gather so it can be lowered to TOSA.GATHER via the backend dialect.
2117
2218 This pass is intended to run only for nodes already gated by GatherSupported.
2319
24- Behavior:
25- - Reshape x from [N,K] to [N,K,1] so values matches TOSA gather's [N,K,C].
26- - Keep indices as [N,W]
27- - Lower using tosa.GATHER.default, producing [N,W,1].
28- - Reshape output to [N,W].
29- - Only insert bool<->int8 casts when x is bool:
20+ Generic behavior:
21+ - Only insert bool<->int8 casts when x is bool:
3022 * If x is bool: gather runs on int8 and output is cast back to bool.
3123 * If x is not bool: gather runs on original dtype and output keeps dtype.
24+
25+ 2D behavior:
26+ - Reshape x from [N,K] to [N,K,1] so values matches TOSA gather's [N,K,C].
27+ - Reshape indices to [N,W].
28+ - Lower using tosa.GATHER.default, producing [N,W,1].
29+ - Reshape output to [N,W].
30+
31+ 3D behavior:
32+ - Permute and reshape x from [N,K,C] to [N*C,K,1].
33+ - Permute and reshape indices from [N,W,C] to [N*C,W].
34+ - Lower using tosa.GATHER.default, producing [N*C,W,1].
35+ - Reshape and permute output to [N,W,C].
36+ - Note that this decomposition requires the channel size of the indices to be the same as the channel
37+ size of the input, which is not guaranteed in Pytorch. We reject nodes not fulfilling this when partitioning.
3238 """
3339
3440 _passes_required_after : Set [Type [ExportPass ]] = set ()
@@ -45,15 +51,17 @@ def call_operator(self, op, args, kwargs, meta):
4551 # GatherSupported should have gated this already; treat violations as errors.
4652 x_shape = x .data .shape
4753 index_shape = index .data .shape
54+ dim = dim % len (x_shape )
4855 if not (
49- dim in (1 , - 1 )
50- and len (x_shape ) == 2
51- and len (index_shape ) == 2
56+ dim in (1 ,)
57+ and len (x_shape ) in ( 2 , 3 )
58+ and len (index_shape ) in ( 2 , 3 )
5259 and index_shape [0 ] == x_shape [0 ]
5360 ):
5461 raise RuntimeError (
55- f"[{ op } ] Unexpected gather pattern; expected "
56- f"x:[N,K], index:[N,W], dim in {{1,-1}}, matching N. "
62+ f"[{ op } ] Unexpected gather pattern; expected one of: "
63+ f"x:[N,K], index:[N,W], dim in {{1,}}, matching N, "
64+ f"x:[N,K,C], index:[N,W,C], dim in {{1,}}, matching N. "
5765 f"Got dim={ dim } , x.shape={ x_shape } , index.shape={ index_shape } ."
5866 )
5967
@@ -62,6 +70,7 @@ def call_operator(self, op, args, kwargs, meta):
6270
6371 view_op = exir_ops .edge .aten .view_copy .default
6472 to_copy_op = exir_ops .edge .dim_order_ops ._to_dim_order_copy .default
73+ permute_copy_op = exir_ops .edge .aten .permute_copy .default
6574
6675 # Use backend dialect gather:
6776 # values: [N,K,C]
@@ -83,40 +92,111 @@ def call_operator(self, op, args, kwargs, meta):
8392 )
8493
8594 # [N,K] -> [N,K,1]
86- values_3d = super ().call_operator (
87- view_op ,
88- (values_in , [N , K , 1 ]),
89- {},
90- meta ,
91- updated = True ,
92- )
93-
94- # indices stays [N,W]
95- gathered_3d = super ().call_operator (
96- tosa_gather_op ,
97- (values_3d , index ),
98- {},
99- meta ,
100- updated = True ,
101- )
102-
103- # [N,W,1] -> [N,W]
104- gathered_2d = super ().call_operator (
105- view_op ,
106- (gathered_3d , [N , W ]),
107- {},
108- meta ,
109- updated = True ,
110- )
95+ if len (x_shape ) < 3 :
96+ values_3d = super ().call_operator (
97+ view_op ,
98+ (values_in , [N , K , 1 ]),
99+ {},
100+ meta ,
101+ updated = True ,
102+ )
103+
104+ # indices stays [N,W]
105+ gathered_3d = super ().call_operator (
106+ tosa_gather_op ,
107+ (values_3d , index ),
108+ {},
109+ meta ,
110+ updated = True ,
111+ )
112+ else :
113+ # For 3D inputs we need to reshuffle both input and indices so that
114+ # the payload is accessed from the batch rather than the channels
115+ # since torch allows for accessing of different indices across channels
116+ # while TOSA does not.
117+ # 1. Permute input [N,K,C] to [N,C,K].
118+ # 2. Rehsape input [N,K,C] to [N*C,K,1].
119+ # 3. Permute indices [N,W,C] to [N,C,W].
120+ # 4. Reshape indices [N,C,W] to [N*C,W].
121+ # 5. Gather using tosa.GATHER with reshaped inputs and indices.
122+ # 6. Reshape output [N*C,W,1] to [N,C,W].
123+ # 7. Permute output [N,C,W] to [N,W,C].
124+
125+ # TODO if index input is a repeat across last dim with C repeats
126+ # we can remove the repeat and directly lower to tosa.GATHER.
127+ values_3d = values_in
128+ N , K , C = x_shape
129+ _ , W , _ = index_shape
130+ values_3d_permuted = super ().call_operator (
131+ permute_copy_op ,
132+ (values_3d , [0 , 2 , 1 ]),
133+ {},
134+ meta ,
135+ updated = True ,
136+ ) # [N,C,K]
137+ values_3d_reshaped = super ().call_operator (
138+ view_op ,
139+ (values_3d_permuted , [N * C , K , 1 ]),
140+ {},
141+ meta ,
142+ updated = True ,
143+ ) # [N*C,K,1]
144+ indices_permuted = super ().call_operator (
145+ permute_copy_op ,
146+ (index , [0 , 2 , 1 ]),
147+ {},
148+ meta ,
149+ updated = True ,
150+ ) # [N,L,W]
151+ indices_reshaped = super ().call_operator (
152+ view_op ,
153+ (indices_permuted , [N * C , W ]),
154+ {},
155+ meta ,
156+ updated = True ,
157+ ) # [N*L,W]
158+ gathered = super ().call_operator (
159+ tosa_gather_op ,
160+ (values_3d_reshaped , indices_reshaped ),
161+ {},
162+ meta ,
163+ updated = True ,
164+ ) # [N*L,W,1]
165+ gathered_reshaped = super ().call_operator (
166+ view_op ,
167+ (gathered , [N , C , W ]),
168+ {},
169+ meta ,
170+ updated = True ,
171+ ) # [N,L,W]
172+ gathered_3d = super ().call_operator (
173+ permute_copy_op ,
174+ (gathered_reshaped , [0 , 2 , 1 ]),
175+ {},
176+ meta ,
177+ updated = True ,
178+ ) # [N,W,L]
179+
180+ if len (x_shape ) == 2 :
181+ # [N,W,1] -> [N,W]
182+ gathered = super ().call_operator (
183+ view_op ,
184+ (gathered_3d , [N , W ]),
185+ {},
186+ meta ,
187+ updated = True ,
188+ )
189+ else :
190+ gathered = gathered_3d
111191
112192 # int8 -> bool (only if needed)
113193 if needs_bool_cast :
114194 return super ().call_operator (
115195 to_copy_op ,
116- (gathered_2d ,),
196+ (gathered ,),
117197 {"dtype" : torch .bool },
118198 meta ,
119199 updated = True ,
120200 )
121201
122- return gathered_2d
202+ return gathered
0 commit comments