Skip to content

Commit 170677f

Browse files
authored
NXP backend: Condition Linear+BatchNorm fusing passes to valid combination only (pytorch#17736)
### Summary Introduce shape conditioning to Linear+BN fusion related passes. Fix `RemoveSimulatedLinearBatchNormFusionQATPass` pass recompiling graph when not necessary. **Problem description:** Linear and BatchNorm layers are fusable only in cases where both layers work with the same dimension. Linear layer works strictly with only the last dimension of it's input. BatchNorm on the other hand works in channels dimension (the second one in most cases) except for one case when using BatchNorm1d and input shape `[N, C]` (see [official PyTorch documentation](https://docs.pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html) for more info). In this one case both BatchNorm and Linear works on the same dimension and are correctly fusable. **Example:** ```python inp = torch.randn((2, 2, 3)) # (N, C, L) lin = nn.Linear(in_features=3, out_features=5) # Weight of shape (5, 3) bn = nn.BatchNorm2d(num_features=2) # Running var shape (2, ) # With the following fusing equation: # linear_w_fused = linear_w * (gamma / sqrt(var + eps)) # And the following shapes: # linear_w.shape: (5, 3); var.shape: (2, ) # We cannot correctly perform the multiplication. ``` ### Test plan New unit test cases were added + relevant older ones were adjusted.
1 parent 86c087d commit 170677f

8 files changed

Lines changed: 321 additions & 193 deletions

File tree

‎backends/nxp/aten_passes/fuse_batch_norm_with_linear_pass.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
# LICENSE file in the root directory of this source tree.
55

66
import torch
7+
8+
from executorch.backends.nxp.backend.graph_utils import get_output_shape
79
from torch.export.unflatten import _assign_attr, _AttrKind
810
from torch.fx import GraphModule, Node
911
from torch.fx.passes.infra.pass_base import PassBase, PassResult
@@ -102,6 +104,14 @@ def _is_linear(node_: Node):
102104
continue # Something other than a Linear node comes before the BatchNorm.
103105

104106
linear_node = bn_node.args[0]
107+
linear_out_shape = get_output_shape(linear_node)
108+
109+
# Skip cases where batch norm does not operate on the same dim as linear.
110+
# Linear operates on the last dim while batch norm works with the first dim right after batch.
111+
# The only case where this occurs is when input shape is (B, C).
112+
if not linear_out_shape or len(linear_out_shape) != 2:
113+
continue
114+
105115
linear_weight_node_or_fq = linear_node.args[1]
106116
linear_bias_node_or_fq = (
107117
linear_node.args[2] if len(linear_node.args) > 2 else None

‎backends/nxp/aten_passes/simulated_linear_bn_fusion_passes/add_simulated_linear_bn_fusion_qat_pass.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
from executorch.backends.nxp.aten_passes.fuse_batch_norm_with_linear_pass import (
1010
_unwrap_if_fq,
1111
)
12-
from executorch.backends.nxp.backend.graph_utils import is_batch_norm, is_op_node
12+
from executorch.backends.nxp.backend.graph_utils import (
13+
get_output_shape,
14+
is_batch_norm,
15+
is_op_node,
16+
)
1317
from torch.fx import GraphModule, Node
1418
from torch.fx.passes.infra.pass_base import PassBase, PassResult
1519
from torchao.quantization.pt2e.export_utils import WrapperModule
@@ -260,6 +264,13 @@ def call(self, graph_module: GraphModule) -> PassResult | None:
260264
continue
261265

262266
linear_node = bn_in
267+
linear_out_shape = get_output_shape(linear_node)
268+
269+
# Skip cases where batch norm does not operate on the same dim as linear.
270+
# Linear operates on the last dim while batch norm works with the first dim right after batch.
271+
# The only case where this occurs is when input shape is (B, C).
272+
if not linear_out_shape or len(linear_out_shape) != 2:
273+
continue
263274

264275
linear_w_node_or_fq = linear_node.args[1]
265276
linear_b_node_or_fq = (

‎backends/nxp/aten_passes/simulated_linear_bn_fusion_passes/remove_simulated_linear_bn_fusion_qat_pass.py‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ def _is_denorm_pattern(node: Node) -> bool:
7878
return False
7979

8080

81-
def _remove_pattern_from_graph(graph_module: GraphModule, pattern: GraphModule):
81+
def _remove_pattern_from_graph(graph_module: GraphModule, pattern: GraphModule) -> bool:
8282
matcher = SubgraphMatcher(
8383
pattern.graph,
8484
match_output=False,
@@ -88,6 +88,7 @@ def _remove_pattern_from_graph(graph_module: GraphModule, pattern: GraphModule):
8888
)
8989
matches: list[InternalMatch] = matcher.match(graph_module.graph, node_name_match="")
9090

91+
made_changes = False
9192
for match in matches:
9293
last_pattern_node = match.anchors[0]
9394
last_matched_subgraph_node = match.nodes_map[last_pattern_node]
@@ -98,31 +99,34 @@ def _remove_pattern_from_graph(graph_module: GraphModule, pattern: GraphModule):
9899
weight = match.placeholder_nodes[0]
99100

100101
last_matched_subgraph_node.replace_all_uses_with(weight)
102+
made_changes = True
101103

102104
for node in match.nodes_map.values():
103105
if node not in match.placeholder_nodes:
104106
graph_module.graph.erase_node(node)
105107

108+
return made_changes
106109

107-
def _remove_late_bias_pattern(graph_module: GraphModule, bias_node: Node):
110+
111+
def _remove_late_bias_pattern(graph_module: GraphModule, bias_node: Node) -> bool:
108112
linear_b_users = list(bias_node.users.keys())
109113

110114
if len(linear_b_users) != 2:
111-
return
115+
return False
112116

113117
if _is_zeros_like(linear_b_users[0]):
114118
zeros_node, maybe_reshape_node = linear_b_users
115119
elif _is_zeros_like(linear_b_users[1]):
116120
maybe_reshape_node, zeros_node = linear_b_users
117121
else:
118-
return
122+
return False
119123

120124
if _is_reshape(maybe_reshape_node):
121125
reshape_node = maybe_reshape_node
122126
reshape_users = list(reshape_node.users.keys())
123127

124128
if len(reshape_users) != 1:
125-
return
129+
return False
126130

127131
add_node = reshape_users[0]
128132
else:
@@ -131,7 +135,7 @@ def _remove_late_bias_pattern(graph_module: GraphModule, bias_node: Node):
131135
add_node = maybe_reshape_node
132136

133137
if not _is_add(add_node):
134-
return
138+
return False
135139

136140
# Remove zeroed linear bias
137141
zeros_node.replace_all_uses_with(bias_node)
@@ -144,10 +148,13 @@ def _remove_late_bias_pattern(graph_module: GraphModule, bias_node: Node):
144148
if reshape_node:
145149
graph_module.graph.erase_node(reshape_node)
146150

151+
return True
152+
147153

148-
def _remove_denorm_and_late_bias(graph_module: GraphModule):
154+
def _remove_denorm_and_late_bias(graph_module: GraphModule) -> bool:
149155
named_modules = dict(graph_module.named_modules(remove_duplicate=False))
150156

157+
made_changes = False
151158
for node in graph_module.graph.nodes:
152159
if not _is_linear(node):
153160
continue
@@ -164,7 +171,7 @@ def _remove_denorm_and_late_bias(graph_module: GraphModule):
164171
has_late_bias = _is_zeros_like(linear_bias_fq_or_zeros)
165172

166173
if has_late_bias:
167-
_remove_late_bias_pattern(
174+
made_changes |= _remove_late_bias_pattern(
168175
graph_module, bias_node=linear_bias_fq_or_zeros.args[0]
169176
)
170177

@@ -173,8 +180,11 @@ def _remove_denorm_and_late_bias(graph_module: GraphModule):
173180
if any(is_batch_norm(user) for user in user_node.users.keys()):
174181
user_node.replace_all_uses_with(node)
175182
graph_module.graph.erase_node(user_node)
183+
made_changes = True
176184
break
177185

186+
return made_changes
187+
178188

179189
class RemoveSimulatedLinearBatchNormFusionQATPass(PassBase):
180190
"""
@@ -192,6 +202,7 @@ def call(self, graph_module: GraphModule) -> PassResult | None:
192202
Given a graph of decomposed aten ops, removes nodes corresponding to linear + batch norm fusion.
193203
"""
194204
is_cuda = False
205+
made_changes = False
195206

196207
graph_module.graph.eliminate_dead_code()
197208
graph_module.recompile()
@@ -219,11 +230,16 @@ def call(self, graph_module: GraphModule) -> PassResult | None:
219230
is_cuda=is_cuda,
220231
)
221232

222-
_remove_pattern_from_graph(graph_module, pattern=scale_match_pattern)
223-
_remove_pattern_from_graph(graph_module, pattern=weight_preprocess_pattern)
224-
_remove_denorm_and_late_bias(graph_module)
233+
made_changes |= _remove_pattern_from_graph(
234+
graph_module, pattern=scale_match_pattern
235+
)
236+
made_changes |= _remove_pattern_from_graph(
237+
graph_module, pattern=weight_preprocess_pattern
238+
)
239+
made_changes |= _remove_denorm_and_late_bias(graph_module)
225240

226-
graph_module.graph.eliminate_dead_code()
227-
graph_module.recompile()
241+
if made_changes:
242+
graph_module.graph.eliminate_dead_code()
243+
graph_module.recompile()
228244

229-
return PassResult(graph_module, True)
245+
return PassResult(graph_module, made_changes)

‎backends/nxp/backend/graph_utils.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,14 @@ def is_op_node(node: Node, target_op) -> bool:
3636

3737
def is_batch_norm(node: Node) -> bool:
3838
return is_op_node(node, batch_norm_target_ops)
39+
40+
41+
def get_output_shape(node: Node) -> tuple[torch.Size] | torch.Size | None:
42+
val = node.meta.get("val")
43+
44+
if isinstance(val, torch.Tensor):
45+
return val.shape
46+
elif isinstance(val, tuple):
47+
return tuple([v.shape for v in val])
48+
49+
return None

‎backends/nxp/tests/ir/edge_passes/test_linear_bn_fusing.py‎

Lines changed: 74 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,12 @@
4545
def test_add_simulated_linear_bn_fusing(input_shape, linear_bias):
4646
calibration_inputs = get_random_calibration_inputs(to_model_input_spec(input_shape))
4747
input_sample = calibration_inputs[0]
48-
model = models.LinearBNModule(
49-
in_features=input_shape[-1],
50-
out_features=5,
51-
linear_bias=linear_bias,
48+
model = models.LinearBatchNormModule(
49+
bias=linear_bias,
50+
input_rank=len(input_shape),
51+
fc_in_features=input_shape[-1],
52+
fc_out_features=5,
53+
bn_in_features=5,
5254
)
5355
model.train()
5456
raw_output = model(input_sample[0])
@@ -110,10 +112,12 @@ def test_full_linear_bn_fusing(input_shape, linear_bias):
110112

111113
calibration_inputs = get_random_calibration_inputs(to_model_input_spec(input_shape))
112114
input_sample = calibration_inputs[0]
113-
model = models.LinearBNModule(
114-
in_features=input_shape[-1],
115-
out_features=5,
116-
linear_bias=linear_bias,
115+
model = models.LinearBatchNormModule(
116+
bias=linear_bias,
117+
input_rank=len(input_shape),
118+
fc_in_features=input_shape[-1],
119+
fc_out_features=5,
120+
bn_in_features=5,
117121
)
118122
model.train()
119123
raw_output = model(input_sample[0])
@@ -175,11 +179,13 @@ def test_input_output_graph_equivalence(input_shape, linear_bias, bn_eps):
175179

176180
calibration_inputs = get_random_calibration_inputs(to_model_input_spec(input_shape))
177181
input_sample = calibration_inputs[0]
178-
model = models.LinearBNModule(
179-
in_features=input_shape[-1],
180-
out_features=5,
181-
linear_bias=linear_bias,
182-
bn_eps=bn_eps,
182+
model = models.LinearBatchNormModule(
183+
bias=linear_bias,
184+
input_rank=len(input_shape),
185+
fc_in_features=input_shape[-1],
186+
fc_out_features=5,
187+
bn_in_features=5,
188+
eps=bn_eps,
183189
)
184190
model.eval()
185191

@@ -216,11 +222,13 @@ def test_linear_bn_full_qat_pipeline_conversion(
216222
"The graph currently produces Linear layer without quantized bias which is incorrect."
217223
)
218224

219-
model = models.LinearBNModule(
220-
in_features=input_shape[-1],
221-
out_features=5,
222-
linear_bias=linear_bias,
223-
bn_eps=bn_eps,
225+
model = models.LinearBatchNormModule(
226+
bias=linear_bias,
227+
input_rank=len(input_shape),
228+
fc_in_features=input_shape[-1],
229+
fc_out_features=5,
230+
bn_in_features=5,
231+
eps=bn_eps,
224232
)
225233
model.eval()
226234

@@ -258,3 +266,51 @@ def test_linear_bn_full_qat_pipeline_conversion(
258266
input_data=input_data,
259267
atol=0.0,
260268
)
269+
270+
271+
@pytest.mark.parametrize("input_shape", [(2, 3, 5), (2, 3, 5, 5), (2, 3, 3, 5, 5)])
272+
@pytest.mark.parametrize("linear_bias", [True, False])
273+
@pytest.mark.parametrize("bn_eps", [1e-5, 1e-6])
274+
def test_incompatible_linear_bn_not_fused(mocker, input_shape, linear_bias, bn_eps):
275+
"""
276+
Test cases ensuring Linear+BN are not fused when not compatible for fusion.
277+
Linear+BN are only fusable when using BatchNorm1d and input has 2 dims (N, L).
278+
"""
279+
280+
# TODO: Add pass for quantizing bias node when Linear has bias=False
281+
if not linear_bias:
282+
pytest.skip(
283+
"Linear with bias=False is not yet supported. "
284+
"The graph currently produces Linear layer without quantized bias which is incorrect."
285+
)
286+
287+
model = models.LinearBatchNormModule(
288+
bias=linear_bias,
289+
input_rank=len(input_shape),
290+
fc_in_features=input_shape[-1],
291+
fc_out_features=5,
292+
bn_in_features=input_shape[1],
293+
eps=bn_eps,
294+
)
295+
model.eval()
296+
model(torch.randn(input_shape))
297+
298+
# Run conversion
299+
edge_program = to_quantized_edge_program(
300+
model, input_shape, use_qat=True, use_neutron_for_format_conversion=False
301+
).exported_program()
302+
303+
assert graph_contains_any_of_ops(
304+
graph=edge_program.graph,
305+
ops=[
306+
exir_ops.edge.aten.addmm.default,
307+
exir_ops.edge.aten.linear.default,
308+
],
309+
)
310+
assert graph_contains_any_of_ops(
311+
graph=edge_program.graph,
312+
ops=batch_norm_target_ops,
313+
)
314+
assert not graph_contains_any_of_ops(
315+
edge_program.graph, [torch.ops.higher_order.executorch_call_delegate]
316+
)

0 commit comments

Comments
 (0)