Skip to content

Commit e9e21b5

Browse files
authored
Qualcomm AI Engine Direct - Fold BatchNorm into the preceding conv (pytorch#22994)
### Summary Fold BatchNorm into the preceding conv for QNN, by running the shared `backends/transforms` pass in the to_edge pipeline. In HTP fp16 a standalone BatchNorm amplifies the conv's rounding error by its scale (~100 after depthwise convs): EfficientNetV2-S fp16 on SM8850 goes from 0.1% to 74.25% top-1 (fp32: 74.40%) and runs 30% faster. Quantized graphs are unaffected. ### Test plan `tests/rework/passes/test.py::test_fuse_batch_norm_with_conv`, `test_qnn_delegate.py::test_qnn_conv1d_batch_norm`; EfficientNetV2-S fp16 on a Galaxy S26 Ultra (SM8850), 2000 ImageNetV2 images. cc @cbilgin @psiddh
1 parent c203fc2 commit e9e21b5

9 files changed

Lines changed: 237 additions & 5 deletions

File tree

‎backends/qualcomm/_passes/BUCK‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ fbcode_target(_kind = runtime.python_library,
1313
deps = [
1414
"//executorch/backends/transforms:addmm_mm_to_linear",
1515
"//executorch/backends/transforms:decompose_sdpa",
16+
"//executorch/backends/transforms:fuse_batch_norm_with_conv",
1617
"//executorch/exir/backend:backend_details",
1718
"//executorch/exir/backend:compile_spec_schema",
1819
"//executorch/backends/qualcomm/quantizer:quantizer",

‎backends/qualcomm/_passes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
from .expand_broadcast_tensor_shape import ExpandBroadcastTensorShape
5454
from .fixed_linear_keep_dim import FixedLinearKeepDim
5555
from .fold_qdq import FoldQDQ
56+
from .fuse_batch_norm_with_conv import FuseBatchNormWithConv
5657
from .fuse_consecutive_cast import FuseConsecutiveCast
5758
from .fuse_consecutive_reshape import FuseConsecutiveReshape
5859
from .fuse_consecutive_transpose import FuseConsecutiveTranspose
@@ -125,6 +126,7 @@
125126
ExpandBroadcastTensorShape,
126127
FixedLinearKeepDim,
127128
FoldQDQ,
129+
FuseBatchNormWithConv,
128130
FuseConsecutiveCast,
129131
FuseConsecutiveReshape,
130132
FuseConsecutiveTranspose,

‎backends/qualcomm/_passes/canonicalize_conv.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,14 @@ def call(self, graph_module: torch.fx.GraphModule):
143143
continue
144144

145145
input_node = node.args[ConvParamIdx.INPUT]
146+
filter_node = node.args[ConvParamIdx.WEIGHT]
146147
filter_placeholder_node = (
147-
# FP graph
148-
node.args[ConvParamIdx.WEIGHT]
149-
if node.args[ConvParamIdx.WEIGHT].op == "placeholder"
148+
# FP graph; a get_attr weight comes from an earlier pass
149+
# that rewrote it (e.g. FuseBatchNormWithConv)
150+
filter_node
151+
if filter_node.op in ("placeholder", "get_attr")
150152
# QDQ graph
151-
else node.args[ConvParamIdx.WEIGHT].args[0]
153+
else filter_node.args[0]
152154
)
153155
filter_tensor = (
154156
get_parameter(filter_placeholder_node, self.edge_program)
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright (c) Qualcomm Innovation Center, Inc.
2+
# All rights reserved
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
9+
from executorch.backends.qualcomm._passes.canonicalize_conv import ConvParamIdx
10+
from executorch.backends.transforms.fuse_batch_norm_with_conv import (
11+
FuseBatchNormWithConvPass,
12+
)
13+
from executorch.exir import ExportedProgram
14+
15+
16+
class FuseBatchNormWithConv(FuseBatchNormWithConvPass):
17+
"""Folds BatchNorm into the preceding convolution, except a transposed one.
18+
19+
fuse_conv_bn_weights() scales dim 0 of the weight, which is the output
20+
channel for a regular convolution but the *input* channel for a transposed
21+
one, whose weight is [in, out/groups, *kernel]. Folding there scales the
22+
wrong axis: it raises when in != out, and silently returns wrong values
23+
when they happen to match. Transposed convolutions therefore keep their
24+
standalone BatchNorm.
25+
"""
26+
27+
@staticmethod
28+
def can_fuse(
29+
conv: torch.fx.Node, bn: torch.fx.Node, program: ExportedProgram
30+
) -> bool:
31+
if (
32+
len(conv.args) > ConvParamIdx.TRANSPOSED
33+
and conv.args[ConvParamIdx.TRANSPOSED]
34+
):
35+
return False
36+
return FuseBatchNormWithConvPass.can_fuse(conv, bn, program)

‎backends/qualcomm/_passes/qnn_pass_manager.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
ExpandBroadcastTensorShape,
5757
FixedLinearKeepDim,
5858
FoldQDQ,
59+
FuseBatchNormWithConv,
5960
FuseConsecutiveCast,
6061
FuseConsecutiveReshape,
6162
FuseConsecutiveTranspose,
@@ -151,6 +152,7 @@ def get_default_pass_activations(cls):
151152
(ExpandBroadcastTensorShape, True),
152153
(FixedLinearKeepDim, True),
153154
(FoldQDQ, True),
155+
(FuseBatchNormWithConv, True),
154156
(FuseConsecutiveReshape, True),
155157
(I64toI32, True),
156158
(InsertCastForFpActQuantizedWeight, True),
@@ -297,14 +299,15 @@ def get_passes_dependency_for_capture_program(cls):
297299
],
298300
AnnotateStack: [RemoveRedundancy],
299301
AnnotateUnbind: [RemoveRedundancy],
300-
CanonicalizeConv: [FoldQDQ],
302+
CanonicalizeConv: [FoldQDQ, FuseBatchNormWithConv],
301303
ConvertBmmToMatmul: [RecomposePixelUnshuffle],
302304
ConvertLinearToConv2d: [FoldQDQ],
303305
DecomposeAcos: [RemoveRedundancy],
304306
DecomposeAddmm: [RemoveRedundancy],
305307
DecomposeAny: [RemoveRedundancy],
306308
DecomposeAtan2: [RemoveRedundancy],
307309
DecomposeColIm: [FoldQDQ],
310+
FuseBatchNormWithConv: [FoldQDQ],
308311
FuseConsecutiveReshape: [FoldQDQ],
309312
DecomposePDist: [RemoveRedundancy],
310313
DecomposeDiagonal: [RemoveRedundancy],
@@ -327,6 +330,9 @@ def get_passes_dependency_for_capture_program(cls):
327330
InsertCastForFpActQuantizedWeight: [FoldQDQ, LayoutTransform],
328331
LayoutTransform: [
329332
AnnotateQuantAttrs,
333+
# conv1d is rewritten into conv2d, so it has to be canonicalized
334+
# before the layout annotation the rewritten node inherits
335+
CanonicalizeConv,
330336
ExpandBroadcastTensorShape,
331337
FixedLinearKeepDim,
332338
],
@@ -409,6 +415,9 @@ def get_to_edge_transform_passes(
409415
kwargs = passes_job[p][QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY]
410416
if "edge_program" in kwargs:
411417
kwargs["edge_program"] = exported_program
418+
# the shared backends/transforms passes name it `exported_program`
419+
if "exported_program" in kwargs:
420+
kwargs["exported_program"] = exported_program
412421
if "compiler_specs" in kwargs:
413422
kwargs["compiler_specs"] = compiler_specs
414423
if "skip_node_id_set" in kwargs:

‎backends/qualcomm/tests/rework/passes/test.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,12 @@ def test_fold_qdq(request, kwargs):
313313
FoldQDQ.test(request, kwargs) # noqa: F405
314314

315315

316+
@enumerate_backends()
317+
@repack_pass_fixtures
318+
def test_fuse_batch_norm_with_conv(request, kwargs):
319+
FuseBatchNormWithConv.test(request, kwargs) # noqa: F405
320+
321+
316322
# Test case tests pytorch dtype conversion,
317323
# since LPAI only supports quant type casting, skip LPAI for now.
318324
@enumerate_backends(

‎backends/qualcomm/tests/rework/src/pattern.py‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
import torch
1616

1717
from executorch.backends.qualcomm import _passes
18+
from executorch.backends.qualcomm._passes.fuse_batch_norm_with_conv import (
19+
FuseBatchNormWithConv as FuseBatchNormWithConvQnn,
20+
)
1821

1922
# Also registers torch.ops.qnn_custom.hadamard_transform (asserted in RecomposeHadamard.test).
2023
from executorch.backends.qualcomm.builders.custom_ops import _hadamard_matrix
@@ -3864,6 +3867,137 @@ def test(
38643867
assertions.assert_no_target(gm, dq_targets)
38653868

38663869

3870+
class FuseBatchNormWithConv:
3871+
"""The shared backends/transforms fold, run from the QNN edge pipeline.
3872+
3873+
Left in place, BatchNorm becomes a standalone QNN op whose per-channel
3874+
scale multiplies the convolution's fp16 rounding error on HTP.
3875+
"""
3876+
3877+
class _Conv1dBn(torch.nn.Module):
3878+
def __init__(self, bias=True):
3879+
super().__init__()
3880+
self.conv = torch.nn.Conv1d(4, 8, kernel_size=3, bias=bias)
3881+
self.bn = torch.nn.BatchNorm1d(8)
3882+
self.eval()
3883+
3884+
def forward(self, x):
3885+
return self.bn(self.conv(x))
3886+
3887+
class _DepthwiseConv2dBn(torch.nn.Module):
3888+
def __init__(self):
3889+
super().__init__()
3890+
self.dw = torch.nn.Conv2d(8, 8, 3, groups=8, bias=False)
3891+
self.bn1 = torch.nn.BatchNorm2d(8)
3892+
self.pw = torch.nn.Conv2d(8, 4, 1)
3893+
self.bn2 = torch.nn.BatchNorm2d(4)
3894+
self.eval()
3895+
3896+
def forward(self, x):
3897+
x = torch.nn.functional.silu(self.bn1(self.dw(x)))
3898+
return self.bn2(self.pw(x))
3899+
3900+
class _ConvTranspose2dBn(torch.nn.Module):
3901+
"""A transposed conv must NOT be folded: its weight is
3902+
[in, out/groups, *kernel], so scaling dim 0 hits the wrong axis."""
3903+
3904+
def __init__(self, cin=4, cout=8):
3905+
super().__init__()
3906+
self.conv = torch.nn.ConvTranspose2d(cin, cout, 3)
3907+
self.bn = torch.nn.BatchNorm2d(cout)
3908+
self.eval()
3909+
3910+
def forward(self, x):
3911+
return self.bn(self.conv(x))
3912+
3913+
class _Conv3dBn(torch.nn.Module):
3914+
def __init__(self):
3915+
super().__init__()
3916+
self.conv = torch.nn.Conv3d(4, 8, kernel_size=3)
3917+
self.bn = torch.nn.BatchNorm3d(8)
3918+
self.eval()
3919+
3920+
def forward(self, x):
3921+
return self.bn(self.conv(x))
3922+
3923+
# The pass matches both inference BatchNorm forms in the edge dialect.
3924+
_BN_OPS = {
3925+
exir_ops.edge.aten.native_batch_norm.default,
3926+
exir_ops.edge.aten._native_batch_norm_legit_no_training.default,
3927+
}
3928+
3929+
@staticmethod
3930+
def _seed_batch_norms(module: torch.nn.Module) -> torch.nn.Module:
3931+
"""Small running variances give the large BatchNorm scales that hurt
3932+
HTP fp16; folding them away is the point of the pass."""
3933+
for m in module.modules():
3934+
if isinstance(m, torch.nn.modules.batchnorm._BatchNorm):
3935+
m.running_mean.uniform_(-2, 2)
3936+
m.running_var.uniform_(1e-3, 1e-2)
3937+
return module
3938+
3939+
@staticmethod
3940+
@unpack_pass_fixtures
3941+
def test(
3942+
subtests,
3943+
quantizer,
3944+
compile_spec,
3945+
backend_type: QnnExecuTorchBackendType,
3946+
assertions: Assertions,
3947+
pass_pipeline: PassPipeline,
3948+
):
3949+
target_pass = FuseBatchNormWithConvQnn
3950+
conv = exir_ops.edge.aten.convolution.default
3951+
3952+
cases = (
3953+
("conv1d_bn", FuseBatchNormWithConv._Conv1dBn(), (1, 4, 16), 1),
3954+
(
3955+
"conv1d_bn_no_bias",
3956+
FuseBatchNormWithConv._Conv1dBn(bias=False),
3957+
(1, 4, 16),
3958+
1,
3959+
),
3960+
(
3961+
"depthwise_conv2d_bn",
3962+
FuseBatchNormWithConv._DepthwiseConv2dBn(),
3963+
(1, 8, 16, 16),
3964+
2,
3965+
),
3966+
("conv3d_bn", FuseBatchNormWithConv._Conv3dBn(), (1, 4, 8, 8, 8), 1),
3967+
)
3968+
for msg, module, shape, conv_count in cases:
3969+
with subtests.test(msg=msg):
3970+
gm = pass_pipeline.lower_edge_ep(
3971+
module=FuseBatchNormWithConv._seed_batch_norms(module),
3972+
sample_input=(torch.randn(*shape),),
3973+
target_pass=target_pass,
3974+
backend_type=backend_type,
3975+
compile_spec=compile_spec,
3976+
quantizer=quantizer,
3977+
).graph_module
3978+
# The BatchNorm is gone and the convolutions it fed survive,
3979+
# now carrying the folded weight and bias.
3980+
assertions.assert_no_target(gm, FuseBatchNormWithConv._BN_OPS)
3981+
assertions.assert_target_count(gm, conv, conv_count)
3982+
3983+
# Folding a transposed conv would scale the wrong weight axis, so the
3984+
# pass has to leave its BatchNorm standing.
3985+
with subtests.test(msg="conv_transpose2d_bn_is_skipped"):
3986+
module = FuseBatchNormWithConv._seed_batch_norms(
3987+
FuseBatchNormWithConv._ConvTranspose2dBn()
3988+
)
3989+
gm = pass_pipeline.lower_edge_ep(
3990+
module=module,
3991+
sample_input=(torch.randn(1, 4, 8, 8),),
3992+
target_pass=target_pass,
3993+
backend_type=backend_type,
3994+
compile_spec=compile_spec,
3995+
quantizer=quantizer,
3996+
).graph_module
3997+
assertions.assert_target_count(gm, conv, 1)
3998+
assertions.assert_target_count(gm, FuseBatchNormWithConv._BN_OPS, 1)
3999+
4000+
38674001
class FuseConsecutiveCast:
38684002
class _ConsecutiveCast(torch.nn.Module):
38694003
def forward(self, x):

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,7 @@ def test_qnn_conv1d_batch_norm(self):
585585
for i, module in enumerate(modules):
586586
with self.subTest(i=i):
587587
self.lower_module_and_test_output(module, sample_input)
588+
self.assert_batch_norm_folded(module, sample_input)
588589

589590
def test_qnn_backend_conv2d(self):
590591
modules = [Conv2dSequential(), Conv2dSequential(bias=False)] # noqa: F405
@@ -2929,6 +2930,7 @@ def test_qnn_backend_conv2d_bn_hardtanh_mean(self):
29292930
module = Conv2dBnHardtanhMean() # noqa: F405
29302931
sample_input = (torch.randn(1, 1, 6, 6),)
29312932
self.lower_module_and_test_output(module, sample_input)
2933+
self.assert_batch_norm_folded(module, sample_input)
29322934

29332935
def test_qnn_backend_conv2d_cat(self):
29342936
module = Conv2dCat() # noqa: F405

‎backends/qualcomm/tests/utils.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
import torch
1616
import torchao
1717
from executorch import exir
18+
from executorch.backends.qualcomm._passes.qnn_pass_manager import (
19+
get_qnn_pass_manager_cls,
20+
)
1821
from executorch.backends.qualcomm.builders.node_visitor import dq_ops
1922

2023
from executorch.backends.qualcomm.debugger.qcom_numerical_comparator_sample import (
@@ -36,6 +39,9 @@
3639
QcomChipset,
3740
QnnExecuTorchLpaiTargetEnv,
3841
)
42+
from executorch.backends.qualcomm.serialization.qc_schema_serialize import (
43+
flatbuffer_to_option,
44+
)
3945
from executorch.backends.qualcomm.utils.constants import (
4046
QCOM_DTYPE,
4147
QCOM_PASS_ACTIVATE_KEY,
@@ -44,14 +50,17 @@
4450
QCOM_ZERO_POINT,
4551
)
4652
from executorch.backends.qualcomm.utils.utils import (
53+
generate_qnn_executorch_option,
4754
get_qnn_context_binary_alignment,
4855
get_soc_to_chipset_map,
56+
qnn_edge_config,
4957
to_edge_transform_and_lower_to_qnn,
5058
)
5159
from executorch.devtools import Inspector
5260
from executorch.devtools.inspector._inspector_utils import TimeScale
5361
from executorch.examples.qualcomm.utils import make_output_dir
5462

63+
from executorch.exir import to_edge_transform_and_lower
5564
from executorch.exir.backend.compile_spec_schema import CompileSpec
5665
from executorch.exir.dialects._ops import ops as exir_ops
5766
from executorch.exir.pass_base import ExportPass
@@ -661,6 +670,37 @@ def validate_intermediate_tensor():
661670
callback=validate_heap_profile,
662671
)
663672

673+
def assert_batch_norm_folded(self, module, sample_inputs):
674+
"""In fp16 a surviving BatchNorm multiplies the convolution's rounding
675+
error by its per-channel scale, so it has to be folded into the
676+
preceding convolution before lowering.
677+
678+
Runs the pipeline the lowering runs, without the partitioner: once the
679+
graph is partitioned it holds nothing but the delegate call, so a
680+
BatchNorm inside the payload would go unnoticed.
681+
"""
682+
exported_program = torch.export.export(module, sample_inputs, strict=True)
683+
backend_type = flatbuffer_to_option(
684+
generate_qnn_executorch_option(self.compiler_specs)
685+
).backend_options.backend_type
686+
pass_manager = get_qnn_pass_manager_cls(backend_type)()
687+
edge_prog_mgr = to_edge_transform_and_lower(
688+
{"forward": pass_manager.transform_for_export_pipeline(exported_program)},
689+
transform_passes={
690+
"forward": pass_manager.get_to_edge_transform_passes(
691+
exported_program, compiler_specs=self.compiler_specs
692+
)
693+
},
694+
partitioner=None,
695+
compile_config=qnn_edge_config(),
696+
)
697+
survivors = [
698+
node
699+
for node in edge_prog_mgr.exported_program().graph.nodes
700+
if node.op == "call_function" and "batch_norm" in str(node.target)
701+
]
702+
self.assertFalse(survivors, f"BatchNorm survived the fold: {survivors}")
703+
664704
def lower_module_and_test_output(
665705
self,
666706
module: torch.nn.Module,

0 commit comments

Comments
 (0)