Skip to content

Commit ad9902c

Browse files
authored
Arm backend: Fuse quantized activations by setting qmin to zp (pytorch#21974)
- Set qmin to the output zero point before removing ReLU. - Keep hardtanh fusion gated on max saturation to qmax. - Skip per-channel quantization because qmin is not in args. - Update tests for signed int8 qmin rewrite cases. Signed-off-by: Saoirse Stewart <saoirse.stewart@arm.com>
1 parent d7c5dd4 commit ad9902c

2 files changed

Lines changed: 187 additions & 10 deletions

File tree

‎backends/arm/_passes/fuse_quantized_activation_pass.py‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
FoldAndAnnotateQParamsPass,
1414
)
1515
from executorch.backends.arm._passes.quant_args import QuantArgs
16-
from executorch.backends.arm.constants import Q_OPS
16+
from executorch.backends.arm.constants import QUANT_PER_TENSOR_OP
1717
from executorch.backends.transforms.remove_getitem_op import RemoveGetItemPass
1818
from executorch.exir.dialects._ops import ops as exir_ops
1919
from executorch.exir.pass_base import ExportPass, PassResult
@@ -29,23 +29,32 @@ class FuseQuantizedActivationPass(ArmPass):
2929

3030
@staticmethod
3131
def _is_fuseable_quantized_activation(node: Node):
32-
"""Fuse activations that have a 0 lower bound and quantized with a qmin
33-
zero-point.
34-
"""
32+
"""Fuse activations that have a 0 lower bound."""
3533
is_fuseable = node.target == exir_ops.edge.aten.relu.default
3634
if node.target == exir_ops.edge.aten.hardtanh.default:
3735
min_val = node.args[1]
3836
is_fuseable = min_val == 0
3937

40-
is_quantized = len(node.users) == 1 and next(iter(node.users)).target in Q_OPS
38+
is_quantized = (
39+
len(node.users) == 1
40+
# qmin is an int argument; only copy scalar zero points into it.
41+
and next(iter(node.users)).target == QUANT_PER_TENSOR_OP
42+
)
4143
if is_fuseable and is_quantized:
4244
quant_node = next(iter(node.users))
4345
quant_args = QuantArgs.from_operator(quant_node.target, quant_node.args)
44-
zp = quant_args.zp
45-
qmin = quant_args.qmin
46-
return zp == qmin
47-
else:
48-
return False
46+
if quant_args.per_channel:
47+
return False
48+
if node.target == exir_ops.edge.aten.hardtanh.default:
49+
max_val = node.args[2]
50+
return quant_args.quantize_value(max_val).item() == quant_args.qmax
51+
return True
52+
return False
53+
54+
@staticmethod
55+
def _set_quant_qmin_to_zp(quant_node: Node):
56+
quant_args = QuantArgs.from_operator(quant_node.target, quant_node.args)
57+
quant_node.update_arg(3, quant_args.zp)
4958

5059
@staticmethod
5160
def _is_fuseable_input(node: Node):
@@ -71,6 +80,8 @@ def call(self, graph_module: torch.fx.GraphModule):
7180
if not FuseQuantizedActivationPass._is_fuseable_input(input_node):
7281
continue
7382

83+
quant_node = next(iter(node.users))
84+
FuseQuantizedActivationPass._set_quant_qmin_to_zp(quant_node)
7485
node.replace_all_uses_with(input_node)
7586
graph_module.graph.erase_node(node)
7687
modified = True

‎backends/arm/test/passes/test_fuse_quantized_activation_pass.py‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7+
import copy
8+
from collections import Counter
79
from typing import Tuple
810

911
import torch
1012
from executorch.backends.arm._passes.fuse_quantized_activation_pass import (
1113
FuseQuantizedActivationPass,
1214
)
1315
from executorch.backends.arm.test.tester.test_pipeline import PassPipeline
16+
from executorch.exir.dialects._ops import ops as exir_ops
1417

1518
input_t = Tuple[torch.Tensor]
1619

@@ -30,6 +33,94 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
3033
return self.relu(self.conv(x))
3134

3235

36+
def _activation_and_quant(
37+
target,
38+
activation_args,
39+
scale: float = 0.1,
40+
zp: int = 0,
41+
qmin: int = -128,
42+
qmax: int = 127,
43+
dtype: torch.dtype = torch.int8,
44+
):
45+
graph = torch.fx.Graph()
46+
x = graph.placeholder("x")
47+
conv = graph.call_function(exir_ops.edge.aten.convolution.default, (x,))
48+
activation = graph.call_function(target, (conv, *activation_args))
49+
quant = graph.call_function(
50+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.default,
51+
(activation, scale, zp, qmin, qmax, dtype),
52+
)
53+
return activation, quant
54+
55+
56+
def _activation_and_tensor_qparam_quant(
57+
target,
58+
activation_args,
59+
qmin: int = -128,
60+
qmax: int = 127,
61+
dtype: torch.dtype = torch.int8,
62+
):
63+
graph = torch.fx.Graph()
64+
x = graph.placeholder("x")
65+
scale = graph.placeholder("scale")
66+
zp = graph.placeholder("zp")
67+
conv = graph.call_function(exir_ops.edge.aten.convolution.default, (x,))
68+
activation = graph.call_function(target, (conv, *activation_args))
69+
quant = graph.call_function(
70+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.tensor,
71+
(activation, scale, zp, qmin, qmax, dtype),
72+
)
73+
return activation, quant
74+
75+
76+
def _activation_quant_graph(
77+
target,
78+
activation_args,
79+
scale: float = 0.1,
80+
zp: int = 0,
81+
qmin: int = -128,
82+
qmax: int = 127,
83+
dtype: torch.dtype = torch.int8,
84+
) -> torch.fx.GraphModule:
85+
graph = torch.fx.Graph()
86+
x = graph.placeholder("x")
87+
x.meta["val"] = torch.empty((2, 4))
88+
weight = graph.placeholder("weight")
89+
weight.meta["val"] = torch.empty((4, 4))
90+
bias = graph.placeholder("bias")
91+
bias.meta["val"] = torch.empty(4)
92+
linear = graph.call_function(exir_ops.edge.aten.linear.default, (x, weight, bias))
93+
linear.meta["val"] = torch.empty((2, 4))
94+
activation = graph.call_function(target, (linear, *activation_args))
95+
activation.meta["val"] = torch.empty((2, 4))
96+
quant = graph.call_function(
97+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.default,
98+
(activation, scale, zp, qmin, qmax, dtype),
99+
)
100+
quant.meta["val"] = torch.empty((2, 4), dtype=dtype)
101+
graph.output(quant)
102+
return torch.fx.GraphModule(torch.nn.Module(), graph)
103+
104+
105+
def _assert_fusion_preserves_output(graph_module: torch.fx.GraphModule) -> None:
106+
original = copy.deepcopy(graph_module)
107+
result = FuseQuantizedActivationPass().call(graph_module)
108+
assert result.modified
109+
110+
inputs = (
111+
torch.tensor(
112+
[[-5.0, -0.06, 0.0, 0.04], [1.2, 12.7, 13.0, 20.0]],
113+
),
114+
torch.eye(4),
115+
torch.zeros(4),
116+
)
117+
with torch.no_grad():
118+
expected = original(*inputs)
119+
actual = result.graph_module(*inputs)
120+
121+
torch.testing.assert_close(actual, expected)
122+
123+
33124
def test_fuse_relu_after_conv_quantized() -> None:
34125
"""Existing behavior: ReLU after conv is fused in quantized graph."""
35126
module = ConvRelu()
@@ -46,3 +137,78 @@ def test_fuse_relu_after_conv_quantized() -> None:
46137
pass_list=[FuseQuantizedActivationPass],
47138
)
48139
pipeline.run()
140+
141+
142+
def test_relu_fusion_rewrites_qmin_to_zero_point() -> None:
143+
activation, quant = _activation_and_quant(
144+
exir_ops.edge.aten.relu.default,
145+
(),
146+
)
147+
148+
assert FuseQuantizedActivationPass._is_fuseable_quantized_activation(activation)
149+
FuseQuantizedActivationPass._set_quant_qmin_to_zp(quant)
150+
151+
assert quant.args[3] == 0
152+
assert quant.args[4] == 127
153+
154+
155+
def test_relu_fusion_preserves_quantized_values() -> None:
156+
graph_module = _activation_quant_graph(
157+
exir_ops.edge.aten.relu.default,
158+
(),
159+
zp=-3,
160+
)
161+
162+
_assert_fusion_preserves_output(graph_module)
163+
counts = Counter(
164+
node.target for node in graph_module.graph.nodes if node.op == "call_function"
165+
)
166+
assert counts[exir_ops.edge.aten.relu.default] == 0
167+
168+
169+
def test_relu_with_tensor_qparams_is_not_fuseable() -> None:
170+
activation, quant = _activation_and_tensor_qparam_quant(
171+
exir_ops.edge.aten.relu.default,
172+
(),
173+
)
174+
175+
assert not FuseQuantizedActivationPass._is_fuseable_quantized_activation(activation)
176+
assert quant.args[3] == -128
177+
assert quant.args[4] == 127
178+
179+
180+
def test_hardtanh_with_non_saturating_max_is_not_fuseable() -> None:
181+
activation, quant = _activation_and_quant(
182+
exir_ops.edge.aten.hardtanh.default,
183+
(0.0, 6.0),
184+
)
185+
186+
assert not FuseQuantizedActivationPass._is_fuseable_quantized_activation(activation)
187+
assert quant.args[3] == -128
188+
assert quant.args[4] == 127
189+
190+
191+
def test_hardtanh_with_saturating_max_rewrites_qmin_to_zero_point() -> None:
192+
activation, quant = _activation_and_quant(
193+
exir_ops.edge.aten.hardtanh.default,
194+
(0.0, 13.0),
195+
)
196+
197+
assert FuseQuantizedActivationPass._is_fuseable_quantized_activation(activation)
198+
FuseQuantizedActivationPass._set_quant_qmin_to_zp(quant)
199+
200+
assert quant.args[3] == 0
201+
assert quant.args[4] == 127
202+
203+
204+
def test_saturating_hardtanh_fusion_preserves_quantized_values() -> None:
205+
graph_module = _activation_quant_graph(
206+
exir_ops.edge.aten.hardtanh.default,
207+
(0.0, 13.0),
208+
)
209+
210+
_assert_fusion_preserves_output(graph_module)
211+
counts = Counter(
212+
node.target for node in graph_module.graph.nodes if node.op == "call_function"
213+
)
214+
assert counts[exir_ops.edge.aten.hardtanh.default] == 0

0 commit comments

Comments
 (0)