Skip to content

Commit ef69e29

Browse files
Arm backend: Add TOSA profile dtype check for aten.index.Tensor (pytorch#18498)
Change-Id: Id4dc204f6513dbfcb4cea2493324b66e9a81cba2 cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell Signed-off-by: Yufeng Shi <yufeng.shi@arm.com>
1 parent 7c5d66e commit ef69e29

3 files changed

Lines changed: 82 additions & 25 deletions

File tree

‎backends/arm/operator_support/index_tensor_support.py‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,12 +127,45 @@ def is_node_tosa_supported(
127127

128128
# Usage 2 guard
129129
input_node = ensure_type(torch.fx.Node, node.args[0])
130-
total_vals = math.prod(get_first_fake_tensor(input_node).shape)
130+
input_val = get_first_fake_tensor(input_node)
131+
total_vals = math.prod(input_val.shape)
131132
if total_vals > torch.iinfo(torch.int32).max:
132133
self.reporter.report_reject(
133134
node,
134135
("Value size exceeds int32 range; would overflow flattened indexing."),
135136
)
136137
return False
137138

139+
values_dtype = input_val.dtype
140+
if values_dtype in (torch.bool, torch.int8, torch.int16, torch.int32):
141+
if not tosa_spec.support_integer():
142+
self.reporter.report_reject(
143+
node,
144+
f"{node.target}: dtype {values_dtype} requires INT profile.",
145+
)
146+
return False
147+
elif values_dtype in (torch.float16, torch.float32, torch.bfloat16):
148+
if values_dtype == torch.bfloat16 and not tosa_spec.support_extension(
149+
"bf16"
150+
):
151+
self.reporter.report_reject(
152+
node,
153+
f"{node.target}: dtype {values_dtype} requires bf16 extension.",
154+
)
155+
return False
156+
if not (tosa_spec.support_float() or tosa_spec.support_integer()):
157+
self.reporter.report_reject(
158+
node,
159+
f"{node.target}: dtype {values_dtype} requires FP profile or "
160+
"INT profile (with quantization).",
161+
)
162+
return False
163+
else:
164+
self.reporter.report_reject(
165+
node,
166+
f"{node.target}: unsupported values dtype {values_dtype}; "
167+
"expected bool/int8/int16/int32/float16/bfloat16/float32.",
168+
)
169+
return False
170+
138171
return True

‎backends/arm/test/models/test_T5ForConditionalGeneration_arm.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,6 @@ def forward(self, input_ids, attention_mask, decoder_input_ids):
8080
return model, inputs
8181

8282

83-
@pytest.mark.xfail(
84-
reason="MLETORCH-1916: Don't partition aten.index.Tensor on int tensors in FP-only profile."
85-
)
8683
@pytest.mark.slow
8784
def test_t5_for_conditional_generation_tosa_FP():
8885
prompt = "summarize: studies have shown that owning a dog is good for you"

‎backends/arm/test/ops/test_index_tensor.py‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -243,73 +243,79 @@ def test_index_tensor_tosa_INT_slice(test_data: input_params_slice):
243243

244244

245245
class IndexTensor(torch.nn.Module):
246-
test_data: dict[input_params] = {
247-
"test_2d_1_idx": (torch.rand(5, 2), (torch.arange(5, dtype=torch.int32),)),
248-
"test_2d_1_less_than_max_idx": (
246+
test_data_fp: dict[input_params] = {
247+
"test_2d_1_idx_fp32": (
248+
torch.rand(5, 2),
249+
(torch.arange(5, dtype=torch.int32),),
250+
),
251+
"test_2d_1_less_than_max_idx_fp32": (
249252
torch.rand(5, 2),
250253
(torch.arange(3, dtype=torch.int32),),
251254
),
252-
"test_2d_1_2d_idx": (
255+
"test_2d_1_2d_idx_fp32": (
253256
torch.rand(5, 2),
254257
(torch.randint(5, size=(4, 3), dtype=torch.int32)),
255258
),
256-
"test_2d_2_idx": (
259+
"test_2d_2_idx_fp32": (
257260
torch.rand(5, 2),
258261
(
259262
torch.randint(5, size=(5,), dtype=torch.int32),
260263
torch.randint(2, size=(5,), dtype=torch.int32),
261264
),
262265
),
263-
"test_2d_2_2d_idx_broadcastable": (
266+
"test_2d_2_2d_idx_broadcastable_fp32": (
264267
torch.rand(5, 2),
265268
(
266269
torch.randint(5, size=(5, 3), dtype=torch.int32),
267270
torch.randint(2, size=(1, 3), dtype=torch.int32),
268271
),
269272
),
270-
"test_2d_2_2d_idx_broadcastable_2": (
273+
"test_2d_2_2d_idx_broadcastable_2_fp32": (
271274
torch.rand(5, 2),
272275
(
273276
torch.randint(5, size=(5, 1), dtype=torch.int32),
274277
torch.randint(2, size=(3,), dtype=torch.int32),
275278
),
276279
),
277-
"test_3d_1_idx": (torch.rand(12, 3, 7), (torch.arange(12, dtype=torch.int32),)),
278-
"test_3d_2_idx": (
280+
"test_3d_1_idx_fp32": (
281+
torch.rand(12, 3, 7),
282+
(torch.arange(12, dtype=torch.int32),),
283+
),
284+
"test_3d_2_idx_fp32": (
279285
torch.rand(12, 3, 7),
280286
(
281287
torch.arange(12, dtype=torch.int32),
282288
torch.randint(3, size=(12,), dtype=torch.int32),
283289
),
284290
),
285-
"test_3d_3_idx": (
291+
"test_3d_3_idx_fp32": (
286292
torch.rand(12, 3, 7),
287293
(
288294
torch.arange(12, dtype=torch.int32),
289295
torch.randint(3, size=(12,), dtype=torch.int32),
290296
torch.randint(7, size=(12,), dtype=torch.int32),
291297
),
292298
),
293-
"test_4d_1_idx": (
299+
"test_4d_1_idx_fp32": (
294300
torch.rand(15, 3, 7, 2),
295301
(torch.arange(15, dtype=torch.int32),),
296302
),
297-
"test_4d_2_idx": (
303+
"test_4d_2_idx_fp32": (
298304
torch.rand(15, 3, 7, 2),
299305
(
300306
torch.randint(15, size=(15,), dtype=torch.int32),
301307
torch.randint(3, size=(1,), dtype=torch.int32),
302308
),
303309
),
304-
"test_4d_3_idx": (
310+
"test_4d_3_idx_fp32": (
305311
torch.rand(15, 3, 7, 2),
306312
(
307313
torch.arange(15, dtype=torch.int32),
308314
torch.randint(3, size=(15,), dtype=torch.int32),
309315
torch.randint(7, size=(15,), dtype=torch.int32),
310316
),
311317
),
312-
"test_4d_4_id_broadcastable": (
318+
"test_4d_4_id_broadcastable_fp32": (
313319
torch.rand(15, 3, 7, 2),
314320
(
315321
torch.arange(15, dtype=torch.int32),
@@ -318,29 +324,50 @@ class IndexTensor(torch.nn.Module):
318324
torch.randint(2, size=(15,), dtype=torch.int32),
319325
),
320326
),
321-
"test_1d_rank4_index": (
327+
"test_1d_rank4_index_fp32": (
322328
torch.rand(12),
323329
(torch.randint(12, size=(1, 2, 1, 3), dtype=torch.int32),),
324330
),
325-
"test_2d_rank4_broadcastable_indices": (
331+
"test_2d_rank4_broadcastable_indices_fp32": (
326332
torch.rand(4, 6),
327333
(
328334
torch.randint(4, size=(1, 2, 1, 1), dtype=torch.int32),
329335
torch.randint(6, size=(1, 1, 3, 1), dtype=torch.int32),
330336
),
331337
),
332-
"test_1d_high_rank_index": (
338+
"test_1d_high_rank_index_fp32": (
333339
torch.rand(24),
334340
(torch.randint(24, size=(1, 1, 2, 1, 3), dtype=torch.int32),),
335341
),
336-
"test_2d_high_rank_broadcastable_indices": (
342+
"test_2d_high_rank_broadcastable_indices_fp32": (
337343
torch.rand(4, 5),
338344
(
339345
torch.randint(4, size=(1, 2, 1, 1, 1), dtype=torch.int32),
340346
torch.randint(5, size=(1, 1, 3, 1, 1), dtype=torch.int32),
341347
),
342348
),
343349
}
350+
test_data_int: dict[input_params] = {
351+
"test_2d_1_idx_int32": (
352+
torch.randint(20, size=(5, 2), dtype=torch.int32),
353+
(torch.arange(5, dtype=torch.int32),),
354+
),
355+
"test_4d_3_idx_int32": (
356+
torch.randint(20, size=(15, 3, 7, 2), dtype=torch.int32),
357+
(
358+
torch.arange(15, dtype=torch.int32),
359+
torch.randint(3, size=(15,), dtype=torch.int32),
360+
torch.randint(7, size=(15,), dtype=torch.int32),
361+
),
362+
),
363+
"test_2d_rank4_broadcastable_indices_int32": (
364+
torch.randint(20, size=(4, 6), dtype=torch.int32),
365+
(
366+
torch.randint(4, size=(1, 2, 1, 1), dtype=torch.int32),
367+
torch.randint(6, size=(1, 1, 3, 1), dtype=torch.int32),
368+
),
369+
),
370+
}
344371
test_data_bf16: dict[input_params] = {
345372
"test_2d_1_idx_bf16": (
346373
torch.rand(size=(3, 4), dtype=torch.bfloat16),
@@ -406,7 +433,7 @@ def forward(self, input_: torch.Tensor, indices: Tuple[None | torch.Tensor]):
406433

407434
@common.parametrize(
408435
"test_data",
409-
IndexTensor.test_data | IndexTensor.test_data_bf16 | IndexTensor.test_data_fp16,
436+
IndexTensor.test_data_fp | IndexTensor.test_data_bf16 | IndexTensor.test_data_fp16,
410437
)
411438
def test_index_tensor_tosa_FP(test_data: input_params):
412439
test_input = test_data
@@ -424,7 +451,7 @@ def test_index_tensor_tosa_FP(test_data: input_params):
424451
)
425452

426453

427-
@common.parametrize("test_data", IndexTensor.test_data)
454+
@common.parametrize("test_data", IndexTensor.test_data_int | IndexTensor.test_data_fp)
428455
def test_index_tensor_tosa_INT(test_data: input_params):
429456
test_input = test_data
430457
with torch.no_grad():
@@ -488,7 +515,7 @@ def test_index_tensor_tosa_INT_none(test_data: input_params):
488515
)
489516

490517

491-
@common.parametrize("test_data", IndexTensor.test_data)
518+
@common.parametrize("test_data", IndexTensor.test_data_int | IndexTensor.test_data_fp)
492519
@common.XfailIfNoCorstone300
493520
def test_index_tensor_u55_INT_not_delegated(test_data: input_params):
494521
"""Ethos-U55 backend BI pipeline test for index.Tensor."""

0 commit comments

Comments
 (0)