Skip to content

Commit 2ec218c

Browse files
NXP backend: Enable MM and AddMM with new Neutron flow. (pytorch#20278)
### Summary This PR enables the `MM` and `AddMM` operators in the NXP backend using the new Neutron C MLIR flow. ### Test plan Unit tests provided. cc @robert-kalmar @JakeStevens @digantdesai @rascani
1 parent 539bda2 commit 2ec218c

19 files changed

Lines changed: 522 additions & 224 deletions

‎backends/nxp/backend/edge_helper.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,43 @@ def node_is_effectively_static_tensor(
109109
)
110110

111111

112+
def weights_are_effectively_static(
113+
node: Node, parameters_mapping: dict[str, Parameter], weight_index: int = 1
114+
) -> bool:
115+
"""Neutron IR sometimes requires some weights to be static. This method checks if this is the case for the
116+
provided `node`.
117+
118+
Sometimes a `permute_copy` is inserted to transpose the weights during edge lowering. The `permute_copy` is
119+
then removed during conversion to Neutron IR if it transposes static data. In those cases, the weights will be
120+
static. Therefore, it is ok if the weights are produced by a `permute_copy` with a static input.
121+
122+
:param node: Tensor node to check for data.
123+
:param parameters_mapping: Dict mapping tensor names to their static data. Should be inferred from the
124+
`state_dict` attribute of an edge program.
125+
:param weight_index: Index to the `node.args` where the weight is located. Defaults to 1.
126+
:return: True if the weight at the given index is effectively static.
127+
"""
128+
129+
def _is_permute_copy(node_: Node) -> bool:
130+
return hasattr(node_, "target") and node_.target == PermuteCopy
131+
132+
if (
133+
_is_dequantize(dq_node := node.args[weight_index])
134+
and _is_quantize(q_node := dq_node.args[0])
135+
and _is_permute_copy(permute_copy_node := q_node.args[0])
136+
):
137+
# The weights are produced by a `permute_copy`. Its input (the weights) must be static.
138+
return node_is_effectively_static_tensor(
139+
permute_copy_node.args[0], parameters_mapping
140+
)
141+
142+
else:
143+
# There is no `permute_copy`. The weights must be static directly.
144+
return node_is_effectively_static_tensor(
145+
node.args[weight_index], parameters_mapping
146+
)
147+
148+
112149
def try_get_tensor_constant_from_node(
113150
graph_module: GraphModule, node: Node
114151
) -> Parameter | None:

‎backends/nxp/backend/ir/converter/node_converter.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,9 @@ def _has_shared_q_params_if_quantized(node: Node) -> bool:
253253

254254
@staticmethod
255255
def is_node_alone_in_partition(
256-
node: Node, partition_list: list[Partition], filter_fn: Callable[[Node], bool]
256+
node: Node,
257+
partition_list: list[Partition],
258+
filter_fn: Callable[[Node], bool] = is_not_qdq_node,
257259
) -> bool:
258260
"""Return True if `node` is the only node in its partition for which `filter_fn`
259261
returns True.
@@ -264,9 +266,8 @@ def is_node_alone_in_partition(
264266
265267
:param node: The torch.fx.Node to check.
266268
:param partition_list: List of proposed partitions.
267-
:param filter_fn: Predicate applied to nodes in the partition.
268-
`node` is considered alone if it is the only node
269-
for which this predicate returns True.
269+
:param filter_fn: Predicate applied to nodes in the partition. `node` is considered alone if it is the only node
270+
for which this predicate returns True. By default, Q/Dq nodes are ignored.
270271
"""
271272
partitions = [p for p in partition_list if node in p.nodes]
272273
if len(partitions) != 1:

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/addmm_converter.py‎

Lines changed: 83 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
1-
# Copyright 2024-2025 NXP
1+
# Copyright 2024-2026 NXP
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
from executorch.backends.nxp.backend.edge_helper import input_rank
6+
import torch
7+
8+
from executorch.backends.nxp.backend.edge_helper import (
9+
input_rank,
10+
node_is_effectively_static_tensor,
11+
weights_are_effectively_static,
12+
)
713
from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList
814
from executorch.backends.nxp.backend.ir.converter.node_converter import (
915
CustomDelegationOptions,
@@ -12,10 +18,18 @@
1218
from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import (
1319
fully_connected_options,
1420
)
21+
22+
from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec
1523
from torch.fx import Node
1624
from torch.nn import Parameter
1725

1826

27+
# The edge operator signature is: aten.addmm(bias, input, weight, *, beta=1, alpha=1)
28+
MAIN_INPUT_IDX = 1
29+
WEIGHT_IDX = 2
30+
BIAS_IDX = 0
31+
32+
1933
class AddMMConverter(NodeConverter):
2034
"""Convert the `aten.addmm` operator to TFLite `FullyConnected` with a bias input."""
2135

@@ -29,12 +43,76 @@ def _is_supported_in_IR(
2943
return False
3044

3145
# The weights must be 2D.
32-
if input_rank(node, 2) != 2:
46+
if input_rank(node, WEIGHT_IDX) != 2:
47+
return False
48+
49+
alpha, beta = node.kwargs.get("alpha", 1), node.kwargs.get("beta", 1)
50+
if alpha != 1 or beta != 1:
51+
# As these cases seem rare, conversion is not implemented for the time being.
52+
return False
53+
54+
# The `aten.addmm` operator allows any bias shape, as long as it is broadcastable with the result of the matrix
55+
# multiplication. That means it supports 4 different shapes: [N, P], [1, P], [P], [1] (provided the MM result
56+
# has shape [N, P]). Out of these 4, Neutron IR allows only [1, P] and [P], both of which are supported on
57+
# Neutron.
58+
bias_shape = list(node.args[BIAS_IDX].meta["val"].shape)
59+
_, p = node.meta["val"].shape
60+
if bias_shape not in [[1, p], [p]]:
61+
return False
62+
63+
return True
64+
65+
@staticmethod
66+
def _is_supported_on_target(
67+
node: Node,
68+
neutron_target_spec: NeutronTargetSpec,
69+
parameters_mapping: dict[str, Parameter],
70+
custom_delegation_options: CustomDelegationOptions,
71+
) -> bool:
72+
# Main input and output must be `int8` or `uint8`.
73+
if not NodeConverter.uses_quantization_type_for_io(
74+
node, [torch.int8, torch.uint8], [MAIN_INPUT_IDX], [0]
75+
):
76+
return False
77+
78+
# Weights must be `int8`.
79+
if not NodeConverter.uses_quantization_type_for_io(
80+
node, [torch.int8], [WEIGHT_IDX], []
81+
):
82+
return False
83+
84+
# Bias must be `int32`.
85+
if not NodeConverter.uses_quantization_type_for_io(
86+
node, [torch.int32], [BIAS_IDX], []
87+
):
88+
return False
89+
90+
# Weights must be constant.
91+
if not weights_are_effectively_static(
92+
node, parameters_mapping, weight_index=WEIGHT_IDX
93+
):
94+
return False
95+
96+
# The bias must be constant.
97+
if not node_is_effectively_static_tensor(
98+
node.args[BIAS_IDX], parameters_mapping
99+
):
33100
return False
34101

35102
return True
36103

37104
def convert(self, node: Node):
105+
"""Convert the `aten.addmm` operator to NeutronIR `FullyConnected`.
106+
The schema is:
107+
addmm(
108+
Tensor self,
109+
Tensor mat1,
110+
Tensor mat2,
111+
*,
112+
Scalar beta=1,
113+
Scalar alpha=1
114+
) -> Tensor
115+
"""
38116
self.assert_convertible(node)
39117

40118
t_op = self._create_tflite_op_with_io_tensors(node)
@@ -47,14 +125,14 @@ def convert(self, node: Node):
47125
w = t_op.tmp_inputs[2]
48126
y = t_op.tmp_outputs[0]
49127

50-
# Assign the operator its TFLite inputs and outputs
128+
# Assign the operator its Neutron IR inputs and outputs
51129
t_op.tmp_inputs = [x, w, bias]
52130
t_op.tmp_outputs = [y]
53131

54132
ops = OpsList(middle_op=t_op)
55133

56134
# The `aten.addmm` uses main input with shape [M, N] and the weights have the shape [N, O].
57-
# TFLite `FullyConnected` requires the weights to have shape [O, N] (if the main input has shape [M, N]).
135+
# Neutron IR `FullyConnected` requires the weights to have shape [O, N] (if the main input has shape [M, N]).
58136
# Insert a `Transpose` operator to permute the weights to achieve correct conversion. (The `Transpose` will not
59137
# be present in the output model if the weights are static.)
60138
ops.add_pre(self.builder.create_transpose_operator_before(t_op, 1, [1, 0]))

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/clamp_converter.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
_is_dequant_node,
1919
_is_quant_node,
2020
CustomDelegationOptions,
21-
is_not_qdq_node,
2221
NodeConverter,
2322
)
2423
from executorch.backends.nxp.backend.ir.converter.quantization_utils import (
@@ -147,9 +146,7 @@ def supports_partitioning_result(
147146
# and at the same time the node does not satisfy delegation requirements.
148147
# In contrast, ReLUN1To1 and ReLU0To1 are supported and delegated successfully.
149148
if bounds in cls.RELU_COMPATIBLE_BOUNDS.values():
150-
is_alone_in_partition = cls.is_node_alone_in_partition(
151-
node, partition_list, filter_fn=is_not_qdq_node
152-
)
149+
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
153150
if is_alone_in_partition:
154151
# noinspection PyTypeChecker
155152
return is_clamp_preserved_under_quantization(

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/mean_dim_converter.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
)
1414
from executorch.backends.nxp.backend.ir.converter.node_converter import (
1515
CustomDelegationOptions,
16-
is_not_qdq_node,
1716
NodeConverter,
1817
)
1918
from executorch.backends.nxp.backend.ir.converter.node_converters.shared.reduce_utils import (
@@ -42,9 +41,7 @@ def supports_partitioning_result(
4241
dim, keepdim = MeanDimConverter._get_attrs(node)
4342
input_shape = node.args[0].meta["val"].shape
4443

45-
is_alone_in_partition = cls.is_node_alone_in_partition(
46-
node, partition_list, filter_fn=is_not_qdq_node
47-
)
44+
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
4845

4946
if is_alone_in_partition and keepdim and all(input_shape[d] == 1 for d in dim):
5047
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/mm_converter.py‎

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
1-
# Copyright 2024-2025 NXP
1+
# Copyright 2024-2026 NXP
22
#
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
from executorch.backends.nxp.backend.edge_helper import input_rank
6+
import torch
7+
8+
from executorch.backends.nxp.backend.edge_helper import (
9+
input_rank,
10+
weights_are_effectively_static,
11+
)
712
from executorch.backends.nxp.backend.ir.converter.conversion.common import OpsList
813
from executorch.backends.nxp.backend.ir.converter.node_converter import (
914
CustomDelegationOptions,
@@ -12,6 +17,7 @@
1217
from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import (
1318
fully_connected_options,
1419
)
20+
from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec
1521
from torch.fx import Node
1622
from torch.nn import Parameter
1723

@@ -33,8 +39,37 @@ def _is_supported_in_IR(
3339

3440
return True
3541

42+
@staticmethod
43+
def _is_supported_on_target(
44+
node: Node,
45+
neutron_target_spec: NeutronTargetSpec,
46+
parameters_mapping: dict[str, Parameter],
47+
custom_delegation_options: CustomDelegationOptions,
48+
) -> bool:
49+
# Main input and output must be `int8` or `uint8`.
50+
if not NodeConverter.uses_quantization_type_for_io(
51+
node, [torch.int8, torch.uint8], [0], [0]
52+
):
53+
return False
54+
55+
# Weights must be `int8`.
56+
if not NodeConverter.uses_quantization_type_for_io(node, [torch.int8], [1], []):
57+
return False
58+
59+
# Weights must be static.
60+
if not weights_are_effectively_static(node, parameters_mapping):
61+
return False
62+
63+
return True
64+
3665
def convert(self, node: Node):
37-
"""Convert the `aten.mm` operator to TFLite `FullyConnected` without a bias input."""
66+
"""Convert the `aten.mm` operator to Neutron IR `FullyConnected` without a bias input.
67+
The schema is:
68+
mm(
69+
Tensor self,
70+
Tensor mat2
71+
) -> Tensor
72+
"""
3873
self.assert_convertible(node)
3974

4075
t_op = self._create_tflite_op_with_io_tensors(node)
@@ -44,14 +79,14 @@ def convert(self, node: Node):
4479
w = t_op.tmp_inputs[1]
4580
y = t_op.tmp_outputs[0]
4681

47-
# Assign the operator its TFLite inputs and outputs
82+
# Assign the operator its Neutron IR inputs and outputs
4883
t_op.tmp_inputs = [x, w]
4984
t_op.tmp_outputs = [y]
5085

5186
ops = OpsList(middle_op=t_op)
5287

5388
# The `aten.mm` uses main input with shape [M, N] and the weights have the shape [N, O].
54-
# TFLite `FullyConnected` requires the weights to have shape [O, N] (if the main input has shape [M, N]).
89+
# Neutron IR `FullyConnected` requires the weights to have shape [O, N] (if the main input has shape [M, N]).
5590
# Insert a `Transpose` operator to permute the weights to achieve correct conversion. (The `Transpose` will not
5691
# be present in the output model if the weights are static.)
5792
ops.add_pre(self.builder.create_transpose_operator_before(t_op, 1, [1, 0]))

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/permute_copy_converter.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
transposition_is_supported_on_neutron,
2828
)
2929
from torch.fx import Node
30+
from torch.fx.passes.infra.partitioner import Partition
3031
from torch.nn import Parameter
3132

3233
Permutation = list[int]
@@ -383,6 +384,26 @@ def _is_supported_in_IR(
383384

384385
return True
385386

387+
@classmethod
388+
def supports_partitioning_result(
389+
cls,
390+
node: Node,
391+
partition_list: list[Partition],
392+
custom_delegation_options: CustomDelegationOptions,
393+
neutron_target_spec: NeutronTargetSpec,
394+
parameters_mapping: dict[str, Parameter],
395+
) -> bool:
396+
has_static_input = node_is_effectively_static_tensor(
397+
node.args[0], parameters_mapping
398+
)
399+
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
400+
if has_static_input and is_alone_in_partition:
401+
# Transpose with a static input is a no-op on Neutron. If it was the only operator in the partition,
402+
# Neutron Converter would produce and empty graph, so delegation is prohibited.
403+
return False
404+
405+
return True
406+
386407
def convert(self, node: Node):
387408
"""Convert the `aten.permute_copy` operator to TFLite `Transpose`."""
388409
self.assert_convertible(node)

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/relu_converter.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
)
1010
from executorch.backends.nxp.backend.ir.converter.node_converter import (
1111
CustomDelegationOptions,
12-
is_not_qdq_node,
1312
NodeConverter,
1413
Partition,
1514
)
@@ -52,9 +51,7 @@ def supports_partitioning_result(
5251
neutron_target_spec: NeutronTargetSpec,
5352
parameters_mapping: dict[str, Parameter],
5453
) -> bool:
55-
is_alone_in_partition = cls.is_node_alone_in_partition(
56-
node, partition_list, filter_fn=is_not_qdq_node
57-
)
54+
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
5855
if is_alone_in_partition:
5956
return is_clamp_preserved_under_quantization(node)
6057

‎backends/nxp/backend/ir/converter/node_converters/ops_converters/upsample_bilinear2d_converter.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
from executorch.backends.nxp.backend.edge_helper import node_has_well_defined_shape
1010
from executorch.backends.nxp.backend.ir.converter.node_converter import (
1111
CustomDelegationOptions,
12-
is_not_qdq_node,
1312
NodeConverter,
1413
requires_channels_first_format,
1514
)
@@ -37,9 +36,7 @@ def supports_partitioning_result(
3736
) -> bool:
3837
input_shape = node.all_input_nodes[0].meta["val"].shape
3938
output_shape = node.meta["val"].shape
40-
is_alone_in_partition = cls.is_node_alone_in_partition(
41-
node, partition_list, filter_fn=is_not_qdq_node
42-
)
39+
is_alone_in_partition = cls.is_node_alone_in_partition(node, partition_list)
4340

4441
if is_alone_in_partition and input_shape == output_shape:
4542
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the

0 commit comments

Comments
 (0)