Skip to content

Commit 42157d7

Browse files
authored
NXP backend: added support for aten.pad with mode reflect (pytorch#21515)
### Summary Added support for `aten.pad` with mode `reflect` using new Neutron MLIR flow. ### Test plan tests can be manually run using `pytest -c /dev/null backends/nxp/tests/` cc @robert-kalmar @JakeStevens @digantdesai @rascani @MartinPavella
1 parent 3b47321 commit 42157d7

9 files changed

Lines changed: 343 additions & 73 deletions

File tree

‎backends/nxp/backend/edge_program_converter.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
exir_ops.edge.aten.mm.default: MMConverter, # noqa F405
5555
exir_ops.edge.aten.mul.Tensor: MulTensorConverter, # noqa F405
5656
exir_ops.edge.aten.neg.default: NegConverter, # noqa F405
57+
exir_ops.edge.aten.pad.default: PadConverter, # noqa F405
5758
exir_ops.edge.aten.permute_copy.default: PermuteCopyConverter, # noqa F405
5859
exir_ops.edge.aten.prelu.default: PReLUConverter, # noqa F405
5960
exir_ops.edge.aten.relu.default: ReLUConverter, # noqa F405

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@
7676
from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.neg_converter import (
7777
NegConverter,
7878
)
79+
from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.pad_converter import (
80+
PadConverter,
81+
)
7982
from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.permute_copy_converter import (
8083
PermuteCopyConverter,
8184
)
@@ -151,6 +154,7 @@
151154
"MMConverter",
152155
"MulTensorConverter",
153156
"NegConverter",
157+
"PadConverter",
154158
"PermuteCopyConverter",
155159
"PReLUConverter",
156160
"QDQPerChannelDequantizeConverter",
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from typing import Collection
7+
8+
import numpy as np
9+
import torch
10+
from executorch.backends.nxp.backend.custom_delegation_options import (
11+
CustomDelegationOptions,
12+
)
13+
14+
from executorch.backends.nxp.backend.ir.converter.conversion.translator import (
15+
apply_permutation_to,
16+
create_channels_first_to_channels_last_permutation,
17+
)
18+
from executorch.backends.nxp.backend.ir.converter.node_converter import NodeConverter
19+
from executorch.backends.nxp.backend.ir.lib.tflite.MirrorPadMode import MirrorPadMode
20+
from executorch.backends.nxp.backend.ir.tflite_generator import tflite_model
21+
from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import (
22+
mirror_pad_options,
23+
)
24+
from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec
25+
from torch.fx import Node
26+
from torch.nn import Parameter
27+
28+
29+
class PadConverter(NodeConverter):
30+
@staticmethod
31+
def _get_mode(node: Node) -> str:
32+
return node.args[2] if len(node.args) > 2 else "constant"
33+
34+
@staticmethod
35+
def _is_supported_in_IR(
36+
node: Node,
37+
parameters_mapping: dict[str, Parameter],
38+
custom_delegation_options: CustomDelegationOptions,
39+
) -> bool:
40+
mode = PadConverter._get_mode(node)
41+
# `constant` mode is decomposed to `aten.constant_pad_nd`.
42+
# Conversion of `replicate`/`circular` Torch padding to TFLite
43+
# is more complicated and is skipped for now.
44+
if mode != "reflect":
45+
return False
46+
47+
return True
48+
49+
@staticmethod
50+
def _is_supported_on_target(
51+
node: Node,
52+
neutron_target_spec: NeutronTargetSpec,
53+
parameters_mapping: dict[str, Parameter],
54+
custom_delegation_options: CustomDelegationOptions,
55+
) -> bool:
56+
if not NodeConverter.uses_quantization_type_for_io(
57+
node,
58+
supported_types=[torch.int8, torch.uint8],
59+
input_indices=[0],
60+
output_indices=[0],
61+
):
62+
return False
63+
64+
return True
65+
66+
@staticmethod
67+
def _convert_paddings_to_tflite(
68+
paddings: Collection[int], input_tensor: tflite_model.Tensor
69+
) -> list[int]:
70+
# Group `padding` by two elements per list.
71+
paddings_grouped = np.array(paddings).reshape(-1, 2)
72+
73+
# In TFLite, `padding` order is reversed.
74+
paddings_reversed = list(reversed(paddings_grouped))
75+
76+
# Add complementary zero pairs to `padding` to match input tensor rank.
77+
zero_pair_compl = [[0, 0]] * (input_tensor.rank - len(paddings_reversed))
78+
padding_tfl = zero_pair_compl + paddings_reversed
79+
80+
if input_tensor.tensor_format.is_channels_last():
81+
# Permute padding to match tensor format.
82+
to_tflite_perm = create_channels_first_to_channels_last_permutation(
83+
input_tensor.rank
84+
)
85+
padding_tfl = apply_permutation_to(padding_tfl, to_tflite_perm)
86+
87+
return padding_tfl
88+
89+
def convert(self, node: Node):
90+
"""Convert `aten.pad.default` to a TFLite padding operator.
91+
92+
The ExecuTorch schema is:
93+
aten::pad(
94+
Tensor self,
95+
SymInt[] pad,
96+
str mode="constant",
97+
float? value=None
98+
) -> Tensor
99+
100+
Only mode "reflect" is handled in this converter.
101+
Mode "constant" is decomposed to `aten.constant_pad_nd` and handled in its own converter.
102+
Modes "replicate"/"circular" are not handled for now, such conversion to TFLite is not trivial.
103+
"""
104+
self.assert_convertible(node)
105+
106+
t_op = self._create_tflite_op_with_io_tensors(node)
107+
x = t_op.tmp_inputs[0]
108+
109+
paddings = self._convert_paddings_to_tflite(node.args[1], x)
110+
111+
paddings_tensor = self.builder.create_tensor_for_data(
112+
np.asarray(paddings, "int32"), "paddings"
113+
)
114+
115+
t_op.builtin_options = mirror_pad_options.MirrorPad(MirrorPadMode.REFLECT)
116+
t_op.tmp_inputs = [x, paddings_tensor]
117+
118+
self.builder.append_operators([t_op])

‎backends/nxp/neutron_partitioner.py‎

Lines changed: 51 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import logging
99
import operator
1010
from dataclasses import dataclass
11+
from functools import partial
1112
from typing import Callable, final, Mapping
1213

1314
import torch
@@ -227,6 +228,7 @@ def tag_qdq_clusters(self, nodes: list[torch.fx.Node]):
227228
exir_ops.edge.aten.mm.default: MMConverter, # noqa F405
228229
exir_ops.edge.aten.mul.Tensor: MulTensorConverter, # noqa F405
229230
exir_ops.edge.aten.neg.default: NegConverter, # noqa F405
231+
exir_ops.edge.aten.pad.default: PadConverter, # noqa F405
230232
exir_ops.edge.aten.permute_copy.default: PermuteCopyConverter, # noqa F405
231233
exir_ops.edge.aten.prelu.default: PReLUConverter, # noqa F405
232234
exir_ops.edge.aten.relu.default: ReLUConverter, # noqa F405
@@ -360,6 +362,11 @@ def __init__(
360362
self.preserve_ops = preserve_ops or []
361363
self.check_op_support = check_op_support
362364

365+
# dict: exir_op -> converter
366+
self.edge_op_to_converter = {}
367+
for exir_op, converter in supported_ops.items():
368+
self.edge_op_to_converter[exir_op._op] = converter
369+
363370
@staticmethod
364371
def _partition_contains_compute_nodes(
365372
partition: Partition, parameters_mapping: dict[str, Parameter]
@@ -501,57 +508,63 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult:
501508
tagged_exported_program=exported_program, partition_tags=partition_tags
502509
)
503510

511+
def _preserve_ops_filter(
512+
self,
513+
parameters_mapping: dict[str, Parameter],
514+
node: torch.fx.Node,
515+
):
516+
"""Filter for `preserve_ops`. This filter makes `preserve_ops` a candidate list of ops that can be or cannot be preserved,
517+
based on conditions below.
518+
"""
519+
# Make `preserve_ops` a candidate list, otherwise the conditions would be checked for all graph ops.
520+
if node.target not in self.preserve_ops:
521+
return False
522+
523+
# If the converter for the node does not exist, the operator should be decomposed.
524+
# When it does not have a converter, it usually means it is natively decomposed into simpler ops that have a converter.
525+
node_converter = self.edge_op_to_converter.get(node.target)
526+
if node_converter is None:
527+
logger.w(
528+
f"Node of target {str(node.target)} might be decomposed to other edge operators "
529+
+ "because no appropriate NodeConverter exists."
530+
)
531+
return False
532+
533+
# op must be decomposed if it is not delegable
534+
delegable = node_converter.is_supported(
535+
node,
536+
self.neutron_target_spec,
537+
parameters_mapping,
538+
self.custom_delegation_options,
539+
)
540+
541+
# if the node is delegable and the base check_op callable determines the node should not be decomposed,
542+
# then do not decompose
543+
base_check_op = (
544+
True if self.check_op_support is None else self.check_op_support(node)
545+
)
546+
return delegable and base_check_op
547+
504548
def ops_to_not_decompose(
505549
self,
506550
ep: ExportedProgram,
507551
) -> tuple[list[torch._ops.OpOverload], Callable[[torch.fx.Node], bool] | None]:
508552
"""
509-
Method to determine which operators SHOULD NOT be decomposed to edge dialect.
553+
Method to determine which operators SHOULD NOT be decomposed to simpler edge ops.
510554
511-
The method returns:
512-
a list of operators NOT to decompose
513-
optional callable - filter
514-
If the operator is in the list and the evaluation of the callable is True (or the callable is not specified),
515-
the operator should not be decomposed (i.e. it truly belongs to the list). Otherwise, it should be decomposed.
555+
Returns:
556+
List[torch._ops.OpOverload]: a list of ops that should not be decomposed.
557+
Optional[Callable[[torch.fx.Node], bool]]]: an optional filter, called for each node in the
558+
graph, that lets a node be decomposed even though its op is in the list above. A node is kept
559+
(not decomposed) only if the filter returns True for it; if it returns False, the node is decomposed.
516560
"""
517561

518562
# The parameters_mapping will only contain `FakeTensor` objects since the partitioning is done with a "Fake model".
519563
# If a node is selected to not be decomposed and it requires the real static parameters in its 'is_supported_*' methods,
520564
# the state dict from the quantized model will need to be provided here.
521565
parameters_mapping = EdgeProgramToIRConverter.map_inputs_to_parameters(ep)
522-
aten_op_to_converter = {}
523-
for exir_op, converter in supported_ops.items():
524-
aten_op_to_converter[exir_op._op] = converter
525566

526567
# determine node format so it can be checked if the nodes are delegable
527568
NodeFormatInference(ep, only_for_op_support_check=True).identify_node_formats()
528569

529-
def check_op_support_extended(node: torch.fx.Node):
530-
# if the operator should not be preserved, then it should be decomposed
531-
if node.target not in self.preserve_ops:
532-
return False
533-
534-
# if the converter for the node does not exist, the operator should be decomposed
535-
node_converter = aten_op_to_converter.get(node.target)
536-
if node_converter is None:
537-
logger.w(
538-
f"Node of target {str(node.target)} might be decomposed to other edge operators "
539-
+ "because no appropriate NodeConverter exists."
540-
)
541-
return False
542-
543-
delegable = node_converter.is_supported(
544-
node,
545-
self.neutron_target_spec,
546-
parameters_mapping,
547-
self.custom_delegation_options,
548-
)
549-
550-
# if the node is delegable and the base check_op callable determines the node should not be decomposed,
551-
# then do not decompose
552-
base_check_op = (
553-
True if self.check_op_support is None else self.check_op_support(node)
554-
)
555-
return delegable and base_check_op
556-
557-
return self.preserve_ops, check_op_support_extended
570+
return self.preserve_ops, partial(self._preserve_ops_filter, parameters_mapping)

‎backends/nxp/tests/executorch_pipeline.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,11 @@ def to_quantized_edge_program(
216216
)
217217

218218
# List of operators to not decompose during the lowering.
219-
preserve_ops = [torch.ops.aten.prelu.default, torch.ops.aten.hardswish.default]
219+
preserve_ops = [
220+
torch.ops.aten.prelu.default,
221+
torch.ops.aten.pad.default,
222+
torch.ops.aten.hardswish.default,
223+
]
220224
compile_spec = generate_neutron_compile_spec(
221225
target,
222226
intermediates_dir=intermediates_dir,

‎backends/nxp/tests/ir/converter/node_converter/test_constant_pad_nd_converter.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,7 @@
1313
ModelBuilder,
1414
)
1515
from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier
16-
from executorch.backends.nxp.tests.models import (
17-
ConstantPadNDConvModule,
18-
ConstantPadNDModule,
19-
)
16+
from executorch.backends.nxp.tests.models import PadConvModule, PadModule
2017
from executorch.backends.nxp.tests.nsys_testing import lower_run_compare
2118
from executorch.backends.nxp.tests.ops_aliases import ConstantPadND, Convolution
2219
from executorch.backends.nxp.tests.use_qat import * # noqa F403
@@ -78,16 +75,15 @@ def assert_delegated_and_output_shape_equals(
7875
def test__basic_nsys_inference(
7976
self, mocker, request, input_shape, paddings, use_qat
8077
):
81-
# These test cases are also supported by the old flow.
82-
model = ConstantPadNDModule(paddings)
78+
model = PadModule(paddings, "constant")
8379
self.assert_delegated(model, input_shape, mocker, request, use_qat)
8480

8581
def test__channels_padding(self, mocker, request):
8682
input_shape = (2, 4, 6)
8783
# These paddings will be applied to the last dimension, which is the channels as the input is formatless.
8884
paddings = (1, 1)
8985
expected_output_shape = (2, 4, 8) # Padded channels.
90-
model = ConstantPadNDModule(paddings)
86+
model = PadModule(paddings, "constant")
9187

9288
self.assert_delegated_and_output_shape_equals(
9389
model, input_shape, expected_output_shape, mocker, request
@@ -97,7 +93,7 @@ def test__batch_padding(self, mocker, request):
9793
input_shape = (2, 4, 6)
9894
paddings = (0, 0, 0, 0, 1, 1) # Padding applied to the batch dimension.
9995
expected_output_shape = (4, 4, 6) # Padded batch.
100-
model = ConstantPadNDModule(paddings)
96+
model = PadModule(paddings, "constant")
10197

10298
self.assert_delegated_and_output_shape_equals(
10399
model, input_shape, expected_output_shape, mocker, request
@@ -107,7 +103,7 @@ def test__batch_padding(self, mocker, request):
107103
def test__specific_constant(self, mocker, request, constant):
108104
input_shape = (2, 4, 6)
109105
paddings = (1, 1)
110-
model = ConstantPadNDModule(paddings, constant)
106+
model = PadModule(paddings, "constant", constant)
111107
self.assert_delegated(model, input_shape, mocker, request)
112108

113109
@pytest.mark.parametrize(
@@ -119,7 +115,13 @@ def test__specific_constant(self, mocker, request, constant):
119115
],
120116
)
121117
def test__channels_first(self, mocker, request, input_shape, paddings):
122-
model = ConstantPadNDConvModule(paddings)
118+
# compute channels size after padding
119+
if len(paddings) // 2 == len(input_shape) - 1:
120+
conv_in_channels = input_shape[1] + paddings[-1] + paddings[-2]
121+
else:
122+
conv_in_channels = input_shape[1]
123+
124+
model = PadConvModule(conv_in_channels, paddings, "constant")
123125
graph_verifier = DetailedGraphVerifier(
124126
mocker,
125127
expected_delegated_ops={ConstantPadND: 1, Convolution: 1},

0 commit comments

Comments
 (0)