|
| 1 | +# Copyright 2026 Arm Limited and/or its affiliates. |
| 2 | +# |
| 3 | +# This source code is licensed under the BSD-style license found in the |
| 4 | +# LICENSE file in the root directory of this source tree. |
| 5 | + |
| 6 | + |
| 7 | +import math |
| 8 | +from typing import Set, Type |
| 9 | + |
| 10 | +import torch |
| 11 | + |
| 12 | +from executorch.backends.arm._passes import ArmPass |
| 13 | +from executorch.backends.arm._passes.replace_scalar_with_tensor_pass import ( |
| 14 | + ReplaceScalarWithTensorByProfilePass, |
| 15 | +) |
| 16 | +from executorch.exir.dialects._ops import ops as exir_ops |
| 17 | +from executorch.exir.pass_base import ExportPass |
| 18 | + |
| 19 | + |
| 20 | +def _get_unfold_copy_decomposition(op) -> tuple: |
| 21 | + """ |
| 22 | + Returns the decomposition of the given aten.unfold_copy operation into |
| 23 | + its equivalent TOSA-supported operations |
| 24 | +
|
| 25 | + Returns: |
| 26 | + A tuple of operator overloads used by the decomposition. |
| 27 | +
|
| 28 | + Raises: |
| 29 | + RuntimeError: If the provided operator is not supported by this pass. |
| 30 | + """ |
| 31 | + |
| 32 | + if op in DecomposeUnfoldToGatherPass._TARGET_OPS: |
| 33 | + return ( |
| 34 | + exir_ops.edge.dim_order_ops._to_dim_order_copy.default, |
| 35 | + exir_ops.edge.aten.view_copy.default, |
| 36 | + exir_ops.edge.aten.arange.start_step, |
| 37 | + exir_ops.edge.aten.mul.Scalar, |
| 38 | + exir_ops.edge.aten.unsqueeze_copy.default, |
| 39 | + exir_ops.edge.aten.add.Tensor, |
| 40 | + exir_ops.edge.aten.expand_copy.default, |
| 41 | + exir_ops.backend.tosa.GATHER.default, |
| 42 | + exir_ops.edge.aten.permute_copy.default, |
| 43 | + ) |
| 44 | + |
| 45 | + raise RuntimeError(f"Can't get unfold_copy decomposition for op {op}") |
| 46 | + |
| 47 | + |
| 48 | +class DecomposeUnfoldToGatherPass(ArmPass): |
| 49 | + """ |
| 50 | + Decompose unfold_copy with backend tosa.GATHER as the core op, plus other |
| 51 | + TOSA-supported ops to build indices and materialize the output layout. |
| 52 | +
|
| 53 | + Supported op: |
| 54 | + y = x.unfold_copy(dimension=dim, size=C, step=S) |
| 55 | +
|
| 56 | + Requirements: |
| 57 | + - size > 0, step > 0 |
| 58 | + - input rank >= 1 |
| 59 | + - dim may be negative (normalized against rank) |
| 60 | + - the selected dimension length K must satisfy K >= size |
| 61 | + - shape must be static enough to compute: |
| 62 | + U = floor((K - C) / S) + 1 |
| 63 | + (i.e. K, C, S known at export time) |
| 64 | +
|
| 65 | + Input / Output (PyTorch semantics): |
| 66 | + - Input: x has shape [d0, d1, ..., d{r-1}] (rank r) |
| 67 | + - Output: y has shape: |
| 68 | + [d0, ..., d{dim-1}, U, d{dim+1}, ..., d{r-1}, C] |
| 69 | + where U = floor((K - C) / S) + 1 and K = x.size(dim) |
| 70 | +
|
| 71 | + Strategy (single gather): |
| 72 | + - If x is bool: cast to int8 for TOSA gather and cast back at the end. |
| 73 | + - Normalize dimension and reshape x to the 3D "values" tensor expected by |
| 74 | + TOSA gather: |
| 75 | + pre = x.shape[:dim] |
| 76 | + post = x.shape[dim+1:] |
| 77 | + P = prod(pre) (or 1 if empty) |
| 78 | + Q = prod(post) (or 1 if empty) |
| 79 | + K = x.shape[dim] |
| 80 | + values = reshape(x, [P, K, Q]) |
| 81 | + - Build a 1D index vector of length U*C: |
| 82 | + idx2d[u,c] = u*S + c for u in [0..U-1], c in [0..C-1] |
| 83 | + idx1d = reshape(idx2d, [U*C]) |
| 84 | + Then expand across P: |
| 85 | + indices = expand(idx1d[None, :], [P, U*C]) |
| 86 | + - Call backend gather: |
| 87 | + tosa.GATHER(values=[P,K,Q], indices=[P,U*C]) -> [P,U*C,Q] |
| 88 | + - Reshape and permute to match PyTorch unfold layout: |
| 89 | + [P,U*C,Q] -> [*pre, U, *post, C] |
| 90 | + """ |
| 91 | + |
| 92 | + _passes_required_after: Set[Type[ExportPass]] = { |
| 93 | + ReplaceScalarWithTensorByProfilePass, |
| 94 | + } |
| 95 | + |
| 96 | + _TARGET_OPS = { |
| 97 | + exir_ops.edge.aten.unfold_copy.default, |
| 98 | + } |
| 99 | + |
| 100 | + _UnfoldCopyInfo = tuple[ |
| 101 | + torch.Tensor, # x_val (FakeTensor) |
| 102 | + int, # C |
| 103 | + int, # S |
| 104 | + int, # K |
| 105 | + int, # U |
| 106 | + int, # UC |
| 107 | + list[int], # pre |
| 108 | + list[int], # post |
| 109 | + int, # P |
| 110 | + int, # Q |
| 111 | + bool, # needs_bool_cast |
| 112 | + ] |
| 113 | + |
| 114 | + def _compute_unfold_copy_params( |
| 115 | + self, |
| 116 | + x: torch.Tensor, |
| 117 | + dim: int, |
| 118 | + size: int, |
| 119 | + step: int, |
| 120 | + ) -> _UnfoldCopyInfo: |
| 121 | + x_val = x.data # FakeTensor |
| 122 | + x_shape = tuple(x_val.shape) |
| 123 | + rank = len(x_shape) |
| 124 | + |
| 125 | + # Checked by UnfoldCopySupported (programmer error if hit here). |
| 126 | + assert rank >= 1, "unfold_copy requires rank >= 1" |
| 127 | + assert size > 0 and step > 0, "unfold_copy requires size/step > 0" |
| 128 | + |
| 129 | + dim_norm = dim % rank |
| 130 | + C = size |
| 131 | + S = step |
| 132 | + |
| 133 | + K = x_shape[dim_norm] |
| 134 | + assert K >= C, f"unfold_copy requires K>=C (K={K}, C={C})" |
| 135 | + |
| 136 | + U = (K - C) // S + 1 |
| 137 | + UC = U * C |
| 138 | + |
| 139 | + pre = list(x_shape[:dim_norm]) |
| 140 | + post = list(x_shape[dim_norm + 1 :]) |
| 141 | + |
| 142 | + P = math.prod(pre) if pre else 1 |
| 143 | + Q = math.prod(post) if post else 1 |
| 144 | + |
| 145 | + needs_bool_cast = x_val.dtype == torch.bool |
| 146 | + |
| 147 | + return (x_val, C, S, K, U, UC, pre, post, P, Q, needs_bool_cast) |
| 148 | + |
| 149 | + def call_operator(self, op, args, kwargs, meta): |
| 150 | + if op not in self._TARGET_OPS: |
| 151 | + return super().call_operator(op, args, kwargs, meta) |
| 152 | + |
| 153 | + x, dim, size, step = args |
| 154 | + |
| 155 | + ( |
| 156 | + x_val, |
| 157 | + C, |
| 158 | + S, |
| 159 | + K, |
| 160 | + U, |
| 161 | + UC, |
| 162 | + pre, |
| 163 | + post, |
| 164 | + P, |
| 165 | + Q, |
| 166 | + needs_bool_cast, |
| 167 | + ) = self._compute_unfold_copy_params(x, dim, size, step) |
| 168 | + |
| 169 | + ( |
| 170 | + to_copy_op, |
| 171 | + view_op, |
| 172 | + arange_op, |
| 173 | + mul_scalar_op, |
| 174 | + unsqueeze_op, |
| 175 | + add_tensor_op, |
| 176 | + expand_op, |
| 177 | + tosa_gather_op, |
| 178 | + permute_op, |
| 179 | + ) = _get_unfold_copy_decomposition(op) |
| 180 | + |
| 181 | + # Note: |
| 182 | + # bool unfolding (and the bool<->int8 casts) are INT-profile only; |
| 183 | + # FP profile should reject these via the UnfoldCopySupported check. |
| 184 | + # ---- optional bool -> int8 ---- |
| 185 | + x_for_gather = x |
| 186 | + if needs_bool_cast: |
| 187 | + x_for_gather = super().call_operator( |
| 188 | + to_copy_op, |
| 189 | + (x,), |
| 190 | + {"dtype": torch.int8}, |
| 191 | + meta, |
| 192 | + updated=True, |
| 193 | + ) |
| 194 | + |
| 195 | + # ---- values: [P, K, Q] ---- |
| 196 | + values_pkq = super().call_operator( |
| 197 | + view_op, |
| 198 | + (x_for_gather, [P, K, Q]), |
| 199 | + {}, |
| 200 | + meta, |
| 201 | + updated=True, |
| 202 | + ) |
| 203 | + |
| 204 | + # ---- build idx1d of length U*C ---- |
| 205 | + # idx_u: [U] |
| 206 | + idx_u = super().call_operator( |
| 207 | + arange_op, |
| 208 | + (0, U), |
| 209 | + { |
| 210 | + "dtype": torch.int32, |
| 211 | + "device": x_val.device, |
| 212 | + }, |
| 213 | + meta, |
| 214 | + updated=True, |
| 215 | + ) |
| 216 | + # idx_u_scaled: [U] = idx_u * S |
| 217 | + idx_u_scaled = super().call_operator( |
| 218 | + mul_scalar_op, |
| 219 | + (idx_u, S), |
| 220 | + {}, |
| 221 | + meta, |
| 222 | + updated=True, |
| 223 | + ) |
| 224 | + # idx_u2d: [U,1] |
| 225 | + idx_u2d = super().call_operator( |
| 226 | + unsqueeze_op, |
| 227 | + (idx_u_scaled, 1), |
| 228 | + {}, |
| 229 | + meta, |
| 230 | + updated=True, |
| 231 | + ) |
| 232 | + |
| 233 | + # idx_c: [C] |
| 234 | + idx_c = super().call_operator( |
| 235 | + arange_op, |
| 236 | + (0, C), |
| 237 | + { |
| 238 | + "dtype": torch.int32, |
| 239 | + "device": x_val.device, |
| 240 | + }, |
| 241 | + meta, |
| 242 | + updated=True, |
| 243 | + ) |
| 244 | + # idx_c2d: [1,C] |
| 245 | + idx_c2d = super().call_operator( |
| 246 | + unsqueeze_op, |
| 247 | + (idx_c, 0), |
| 248 | + {}, |
| 249 | + meta, |
| 250 | + updated=True, |
| 251 | + ) |
| 252 | + |
| 253 | + # idx2d: [U,C] = idx_u2d + idx_c2d |
| 254 | + idx2d = super().call_operator( |
| 255 | + add_tensor_op, |
| 256 | + (idx_u2d, idx_c2d), |
| 257 | + {}, |
| 258 | + meta, |
| 259 | + updated=True, |
| 260 | + ) |
| 261 | + |
| 262 | + # idx1d: [U*C] |
| 263 | + idx1d = super().call_operator( |
| 264 | + view_op, |
| 265 | + (idx2d, [UC]), |
| 266 | + {}, |
| 267 | + meta, |
| 268 | + updated=True, |
| 269 | + ) |
| 270 | + |
| 271 | + # indices: [P, U*C] by expanding from [1, U*C] |
| 272 | + idx_1xUC = super().call_operator( |
| 273 | + unsqueeze_op, |
| 274 | + (idx1d, 0), |
| 275 | + {}, |
| 276 | + meta, |
| 277 | + updated=True, |
| 278 | + ) |
| 279 | + indices_puc = super().call_operator( |
| 280 | + expand_op, |
| 281 | + (idx_1xUC, [P, UC]), |
| 282 | + {}, |
| 283 | + meta, |
| 284 | + updated=True, |
| 285 | + ) |
| 286 | + |
| 287 | + # ---- backend tosa gather ---- |
| 288 | + # tosa.GATHER(values=[P,K,Q], indices=[P,UC]) -> [P,UC,Q] |
| 289 | + gathered_pucq = super().call_operator( |
| 290 | + tosa_gather_op, |
| 291 | + (values_pkq, indices_puc), |
| 292 | + {}, |
| 293 | + meta, |
| 294 | + updated=True, |
| 295 | + ) |
| 296 | + |
| 297 | + # [P,UC,Q] -> [P,U,C,Q] |
| 298 | + pucq = super().call_operator( |
| 299 | + view_op, |
| 300 | + (gathered_pucq, [P, U, C, Q]), |
| 301 | + {}, |
| 302 | + meta, |
| 303 | + updated=True, |
| 304 | + ) |
| 305 | + |
| 306 | + # Unflatten Q back to post, and P back to pre: |
| 307 | + # current logical layout: [*pre_flat=P, U, C, *post_flat=Q] |
| 308 | + if post: |
| 309 | + # [P,U,C,Q] -> [P,U,C,*post] |
| 310 | + shaped = super().call_operator( |
| 311 | + view_op, |
| 312 | + (pucq, [P, U, C, *post]), |
| 313 | + {}, |
| 314 | + meta, |
| 315 | + updated=True, |
| 316 | + ) |
| 317 | + else: |
| 318 | + # [P,U,C,Q(=1)] -> [P,U,C] |
| 319 | + shaped = super().call_operator( |
| 320 | + view_op, |
| 321 | + (pucq, [P, U, C]), |
| 322 | + {}, |
| 323 | + meta, |
| 324 | + updated=True, |
| 325 | + ) |
| 326 | + |
| 327 | + if pre: |
| 328 | + # [P,U,C,*post] -> [*pre, U, C, *post] |
| 329 | + shaped = super().call_operator( |
| 330 | + view_op, |
| 331 | + (shaped, [*pre, U, C, *post]), |
| 332 | + {}, |
| 333 | + meta, |
| 334 | + updated=True, |
| 335 | + ) |
| 336 | + else: |
| 337 | + # drop P(=1): [P,U,C,*post] -> [U, C, *post] |
| 338 | + shaped = super().call_operator( |
| 339 | + view_op, |
| 340 | + (shaped, [U, C, *post]), |
| 341 | + {}, |
| 342 | + meta, |
| 343 | + updated=True, |
| 344 | + ) |
| 345 | + |
| 346 | + # Move C to the last dimension: |
| 347 | + # from [*pre, U, C, *post] -> [*pre, U, *post, C] |
| 348 | + pre_len = len(pre) |
| 349 | + post_len = len(post) |
| 350 | + rank_cur = pre_len + 1 + 1 + post_len # pre + U + C + post |
| 351 | + c_idx = pre_len + 1 |
| 352 | + |
| 353 | + perm = list(range(0, pre_len + 1)) + list(range(c_idx + 1, rank_cur)) + [c_idx] |
| 354 | + out = super().call_operator( |
| 355 | + permute_op, |
| 356 | + (shaped, perm), |
| 357 | + {}, |
| 358 | + meta, |
| 359 | + updated=True, |
| 360 | + ) |
| 361 | + |
| 362 | + # ---- optional int8 -> bool ---- |
| 363 | + if needs_bool_cast: |
| 364 | + out = super().call_operator( |
| 365 | + to_copy_op, |
| 366 | + (out,), |
| 367 | + {"dtype": torch.bool}, |
| 368 | + meta, |
| 369 | + updated=True, |
| 370 | + ) |
| 371 | + |
| 372 | + return out |
0 commit comments