Skip to content

Commit 08ce4df

Browse files
Cortex-M backend: Let output node inherit qspec from input (pytorch#22805)
Previously the output node would be quantized by the global quantizer, which would lead to duplicate q-dq node pairs for softmax which uses a fixed qspec. The behaviour you typically want for the output is to inherit from the previous node instead. + Add regression testing for the softmax case cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Adrian Lundell <adrian.lundell@arm.com>
1 parent edad0a7 commit 08ce4df

8 files changed

Lines changed: 77 additions & 17 deletions

File tree

‎backends/arm/quantizer/arm_quantizer_utils.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,10 +794,29 @@ def _annotate_shared_cluster(self, root_node: Node) -> None:
794794
)
795795
return
796796

797+
def _annotate_graph_outputs(self, model: torch.fx.GraphModule) -> None:
798+
for output_node in model.graph.nodes:
799+
if output_node.op != "output" or self._is_annotated(output_node):
800+
continue
801+
802+
input_qspec_map = {
803+
input_node: input_node.meta[Q_ANNOTATION_KEY].output_qspec
804+
for input_node in output_node.all_input_nodes
805+
if is_output_annotated(input_node)
806+
}
807+
if input_qspec_map:
808+
_mark_node_as_quantized(
809+
output_node,
810+
input_qspec_map,
811+
None,
812+
is_quantized=True,
813+
)
814+
797815
def annotate(self, model: torch.fx.GraphModule) -> None: # type: ignore[override]
798816
for node in model.graph.nodes:
799817
if node.target in self.targets and not self._is_annotated(node):
800818
self._annotate_shared_cluster(node)
819+
self._annotate_graph_outputs(model)
801820

802821
def validate(self, model: torch.fx.GraphModule) -> bool: # type: ignore[override]
803822
return True

‎backends/cortex_m/quantizer/node_finders.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,18 @@ def __repr__(self) -> str:
3535
return f"{self.__class__.__name__} targeting all nodes"
3636

3737

38+
class GlobalNoOutputNodeFinder(NodeFinder):
39+
"""Finds all graph nodes except the output node. Output nodes
40+
typically wants to inherit the previous qspec rather than applying a new
41+
qspec."""
42+
43+
def find_nodes(self, model: GraphModule) -> Iterator[Node]:
44+
return (n for n in model.graph.nodes if n.op != "output")
45+
46+
def __repr__(self) -> str:
47+
return f"{self.__class__.__name__} targeting all non-output nodes"
48+
49+
3850
class InputNodeFinder(NodeFinder):
3951
"""
4052
Finds all placeholder nodes.

‎backends/cortex_m/quantizer/quantizer.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
1616
from executorch.backends.cortex_m.passes.cortex_m_pass_manager import CortexMPassManager
1717
from executorch.backends.cortex_m.quantizer.node_finders import (
18-
GlobalNodeFinder,
18+
GlobalNoOutputNodeFinder,
1919
NodeTargetNodeFinder,
2020
)
2121
from executorch.backends.cortex_m.quantizer.pattern_matcher import PatternMatcher
@@ -100,7 +100,7 @@ def __init__(
100100
),
101101
PatternQuantizer(
102102
per_tensor_config,
103-
node_finder=GlobalNodeFinder(),
103+
node_finder=GlobalNoOutputNodeFinder(),
104104
pattern_matcher=pattern_matcher,
105105
),
106106
SharedQspecQuantizer(),

‎backends/cortex_m/test/models/test_silero_vad.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@
3636
"executorch_exir_dialects_edge__ops_aten_tanh_default": 2,
3737
"executorch_exir_dialects_edge__ops_aten_unsqueeze_copy_default": 2,
3838
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
39-
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 16,
40-
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 15,
39+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 14,
40+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 13,
4141
}
4242
# The final `sigmoid(final_conv(x))` and the STFT magnitude's sqrt now lower to
4343
# cortex_m.quantized_activation. The 3 remaining sigmoids and 2 tanhs are LSTMCell gates: PyTorch export
@@ -70,8 +70,8 @@
7070
"executorch_exir_dialects_edge__ops_aten_tanh_default": 2,
7171
"executorch_exir_dialects_edge__ops_aten_unsqueeze_copy_default": 2,
7272
"executorch_exir_dialects_edge__ops_aten_view_copy_default": 1,
73-
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 7,
74-
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 7,
73+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 5,
74+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 5,
7575
"executorch_exir_dialects_edge__ops_cortex_m_quantized_activation_default": 2,
7676
"executorch_exir_dialects_edge__ops_cortex_m_quantized_add_default": 1,
7777
}

‎backends/cortex_m/test/ops/test_add.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,14 +79,14 @@ class CortexMAlphaAdd(ModelAlpha):
7979

8080
ops_before_transforms = {
8181
"executorch_exir_dialects_edge__ops_aten_add_Tensor": 1,
82-
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 3,
83-
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 3,
82+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 2,
83+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 2,
8484
}
8585
ops_after_transforms = {
8686
"executorch_exir_dialects_edge__ops_cortex_m_quantized_add_default": 0,
8787
"executorch_exir_dialects_edge__ops_aten_add_Tensor": 1,
88-
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 3,
89-
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 3,
88+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 2,
89+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 2,
9090
}
9191

9292

‎backends/cortex_m/test/ops/test_avg_pool2d.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,12 +174,19 @@ def test_dialect_avg_pool2d_fallback(test_case, cortex_m_target):
174174
tester = CortexMTester(
175175
test_case.model, test_case.example_inputs, target_config=cortex_m_target
176176
)
177+
ops_before = dict(test_case.model.ops_before_transforms)
178+
ops_before.update(
179+
{
180+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 1,
181+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 1,
182+
}
183+
)
177184
tester.test_dialect(
178-
test_case.model.ops_before_transforms,
185+
ops_before,
179186
{
180187
"executorch_exir_dialects_edge__ops_aten_avg_pool2d_default": 1,
181-
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 2,
182-
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 2,
188+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 1,
189+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 1,
183190
},
184191
qtol=1,
185192
)

‎backends/cortex_m/test/ops/test_softmax.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@
66

77
import torch
88
from executorch.backends.arm.test.common import parametrize, xfail_type
9+
from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer
910
from executorch.backends.cortex_m.test.tester import (
1011
CortexMTester,
1112
McuTestCase,
1213
ramp_tensor,
1314
)
15+
from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e
1416

1517

1618
class CortexMSoftmax(torch.nn.Module):
@@ -80,6 +82,26 @@ def test_dialect_softmax(test_case, cortex_m_target):
8082
)
8183

8284

85+
def test_softmax_output_does_not_requantize():
86+
inputs = (ramp_tensor(-8, 8, (4, 8)),)
87+
exported = torch.export.export(CortexMSoftmax(dim=-1), inputs, strict=True)
88+
prepared = prepare_pt2e(exported.module(), CortexMQuantizer())
89+
prepared(*inputs)
90+
converted = convert_pt2e(prepared)
91+
92+
quantize = torch.ops.quantized_decomposed.quantize_per_tensor.default
93+
dequantize = torch.ops.quantized_decomposed.dequantize_per_tensor.default
94+
requantize_nodes = [
95+
node
96+
for node in converted.graph.nodes
97+
if node.target == quantize
98+
and isinstance(node.args[0], torch.fx.Node)
99+
and node.args[0].target == dequantize
100+
and any(user.target == dequantize for user in node.users)
101+
]
102+
assert not requantize_nodes
103+
104+
83105
@parametrize("test_case", test_cases, xfails=xfail_cases_impl)
84106
def test_implementation_softmax(test_case, cortex_m_target):
85107
tester = CortexMTester(

‎backends/cortex_m/test/ops/test_sub.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,14 @@ class CortexMFloatAlphaSub(torch.nn.Module):
8181

8282
ops_before_transforms = {
8383
"executorch_exir_dialects_edge__ops_aten_sub_Tensor": 1,
84-
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 3,
85-
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 3,
84+
"executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 2,
85+
"executorch_exir_dialects_edge__ops_quantized_decomposed_dequantize_per_tensor_default": 2,
8686
}
8787
ops_after_transforms = {
8888
"executorch_exir_dialects_edge__ops_cortex_m_quantized_add_default": 0,
8989
"executorch_exir_dialects_edge__ops_aten_sub_Tensor": 1,
90-
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 3,
91-
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 3,
90+
"executorch_exir_dialects_edge__ops_cortex_m_quantize_per_tensor_default": 2,
91+
"executorch_exir_dialects_edge__ops_cortex_m_dequantize_per_tensor_default": 2,
9292
}
9393

9494
def forward(self, x, y):

0 commit comments

Comments
 (0)