|
| 1 | +# Copyright 2026 NXP |
| 2 | +# |
| 3 | +# This source code is licensed under the BSD-style license found in the |
| 4 | +# LICENSE file in the root directory of this source tree. |
| 5 | + |
| 6 | +from functools import partial |
| 7 | + |
| 8 | +import torch |
| 9 | +from executorch.backends.nxp.aten_passes.fuse_batch_norm_with_linear_pass import ( |
| 10 | + _unwrap_if_fq, |
| 11 | +) |
| 12 | +from executorch.backends.nxp.aten_passes.simulated_linear_bn_fusion_passes.add_simulated_linear_bn_fusion_qat_pass import ( |
| 13 | + _get_compute_scale_factor_pattern, |
| 14 | + _get_linear_weight_preprocess_pattern, |
| 15 | +) |
| 16 | +from executorch.backends.nxp.backend.graph_utils import is_batch_norm, is_op_node |
| 17 | +from torch.fx import GraphModule, Node |
| 18 | +from torch.fx.passes.infra.pass_base import PassBase, PassResult |
| 19 | +from torch.fx.passes.utils.matcher_utils import InternalMatch, SubgraphMatcher |
| 20 | +from torchao.quantization.pt2e.qat_utils import _get_aten_graph_module_for_pattern |
| 21 | + |
| 22 | +_is_add = partial(is_op_node, target_op=torch.ops.aten.add.Tensor) |
| 23 | +_is_div = partial(is_op_node, target_op=torch.ops.aten.div.Tensor) |
| 24 | +_is_linear = partial(is_op_node, target_op=torch.ops.aten.linear.default) |
| 25 | +_is_reshape = partial(is_op_node, target_op=torch.ops.aten.reshape) |
| 26 | +_is_zeros_like = partial(is_op_node, target_op=torch.ops.aten.zeros_like) |
| 27 | + |
| 28 | + |
| 29 | +def _is_denorm_pattern(node: Node) -> bool: |
| 30 | + if not _is_div(node): |
| 31 | + return False |
| 32 | + |
| 33 | + if not hasattr(node, "users"): |
| 34 | + return False |
| 35 | + |
| 36 | + div_users = node.users.keys() |
| 37 | + if len(list(div_users)) < 1: |
| 38 | + return False |
| 39 | + |
| 40 | + if any(is_batch_norm(user) for user in div_users): |
| 41 | + return True |
| 42 | + |
| 43 | + return False |
| 44 | + |
| 45 | + |
| 46 | +def _remove_pattern_from_graph(graph_module: GraphModule, pattern: GraphModule): |
| 47 | + matcher = SubgraphMatcher( |
| 48 | + pattern.graph, |
| 49 | + match_output=False, |
| 50 | + match_placeholder=False, |
| 51 | + remove_overlapping_matches=True, |
| 52 | + ignore_literals=True, |
| 53 | + ) |
| 54 | + matches: list[InternalMatch] = matcher.match(graph_module.graph, node_name_match="") |
| 55 | + |
| 56 | + for match in matches: |
| 57 | + last_pattern_node = match.anchors[0] |
| 58 | + last_matched_subgraph_node = match.nodes_map[last_pattern_node] |
| 59 | + weight = match.placeholder_nodes[0] |
| 60 | + |
| 61 | + last_matched_subgraph_node.replace_all_uses_with(weight) |
| 62 | + |
| 63 | + for node in match.nodes_map.values(): |
| 64 | + if node not in match.placeholder_nodes: |
| 65 | + graph_module.graph.erase_node(node) |
| 66 | + |
| 67 | + |
| 68 | +def _remove_late_bias_pattern(graph_module: GraphModule, bias_node: Node): |
| 69 | + linear_b_users = list(bias_node.users.keys()) |
| 70 | + |
| 71 | + if len(linear_b_users) != 2: |
| 72 | + return |
| 73 | + |
| 74 | + if _is_zeros_like(linear_b_users[0]): |
| 75 | + zeros_node, maybe_reshape_node = linear_b_users |
| 76 | + elif _is_zeros_like(linear_b_users[1]): |
| 77 | + maybe_reshape_node, zeros_node = linear_b_users |
| 78 | + else: |
| 79 | + return |
| 80 | + |
| 81 | + if _is_reshape(maybe_reshape_node): |
| 82 | + reshape_node = maybe_reshape_node |
| 83 | + reshape_users = list(reshape_node.users.keys()) |
| 84 | + |
| 85 | + if len(reshape_users) != 1: |
| 86 | + return |
| 87 | + |
| 88 | + add_node = reshape_users[0] |
| 89 | + else: |
| 90 | + # Handles no reshape node when bias is scalar |
| 91 | + reshape_node = None |
| 92 | + add_node = maybe_reshape_node |
| 93 | + |
| 94 | + if not _is_add(add_node): |
| 95 | + return |
| 96 | + |
| 97 | + # Remove zeroed linear bias |
| 98 | + zeros_node.replace_all_uses_with(bias_node) |
| 99 | + graph_module.graph.erase_node(zeros_node) |
| 100 | + |
| 101 | + # Remove late bias addition |
| 102 | + add_node.replace_all_uses_with(add_node.args[0]) |
| 103 | + graph_module.graph.erase_node(add_node) |
| 104 | + |
| 105 | + if reshape_node: |
| 106 | + graph_module.graph.erase_node(reshape_node) |
| 107 | + |
| 108 | + |
| 109 | +def _remove_denorm_and_late_bias(graph_module: GraphModule): |
| 110 | + named_modules = dict(graph_module.named_modules(remove_duplicate=False)) |
| 111 | + |
| 112 | + for node in graph_module.graph.nodes: |
| 113 | + if not _is_linear(node): |
| 114 | + continue |
| 115 | + |
| 116 | + linear_node = node |
| 117 | + |
| 118 | + if len(linear_node.args) < 2: |
| 119 | + continue |
| 120 | + |
| 121 | + maybe_linear_bias = linear_node.args[2] if len(linear_node.args) > 2 else None |
| 122 | + linear_bias_fq_or_zeros = _unwrap_if_fq( |
| 123 | + maybe_linear_bias, named_modules=named_modules |
| 124 | + ) |
| 125 | + has_late_bias = _is_zeros_like(linear_bias_fq_or_zeros) |
| 126 | + |
| 127 | + if has_late_bias: |
| 128 | + _remove_late_bias_pattern( |
| 129 | + graph_module, bias_node=linear_bias_fq_or_zeros.args[0] |
| 130 | + ) |
| 131 | + |
| 132 | + for user_node in linear_node.users: |
| 133 | + if _is_denorm_pattern(user_node): |
| 134 | + if any(is_batch_norm(user) for user in user_node.users.keys()): |
| 135 | + user_node.replace_all_uses_with(node) |
| 136 | + graph_module.graph.erase_node(user_node) |
| 137 | + break |
| 138 | + |
| 139 | + |
| 140 | +class RemoveSimulatedLinearBatchNormFusionQATPass(PassBase): |
| 141 | + """ |
| 142 | + In order for QAT to work correctly with fused linear + batch norm operators, |
| 143 | + simulated linear + batch norm fusion should be added using AddSimulatedLinearBatchNormFusionQATPass. |
| 144 | +
|
| 145 | + After the QAT training, before inserting QDQ nodes, nodes added by the simulated fusion should be removed. |
| 146 | + This pass removes all artifacts created by AddSimulatedLinearBatchNormFusionQATPass and reverts |
| 147 | + the graph back to the layout before the simulated fusion was applied. |
| 148 | + See `add_simulated_linear_bn_fusion_qat_pass.py` for more details. |
| 149 | + """ |
| 150 | + |
| 151 | + def call(self, graph_module: GraphModule) -> PassResult | None: |
| 152 | + """ |
| 153 | + Given a graph of decomposed aten ops, removes nodes corresponding to linear + batch norm fusion. |
| 154 | + """ |
| 155 | + is_cuda = False |
| 156 | + |
| 157 | + graph_module.graph.eliminate_dead_code() |
| 158 | + graph_module.recompile() |
| 159 | + |
| 160 | + _scale_compute_example_inputs = ( |
| 161 | + torch.randn(1), |
| 162 | + torch.randn(1), |
| 163 | + ) |
| 164 | + _preprocess_example_inputs = ( |
| 165 | + torch.randn(1, 1), |
| 166 | + torch.randn(1), |
| 167 | + ) |
| 168 | + |
| 169 | + scale_pattern = _get_compute_scale_factor_pattern() |
| 170 | + scale_match_pattern = _get_aten_graph_module_for_pattern( |
| 171 | + pattern=scale_pattern, |
| 172 | + example_inputs=_scale_compute_example_inputs, |
| 173 | + is_cuda=is_cuda, |
| 174 | + ) |
| 175 | + |
| 176 | + weight_preprocess_pattern = _get_linear_weight_preprocess_pattern() |
| 177 | + weight_preprocess_pattern = _get_aten_graph_module_for_pattern( |
| 178 | + pattern=weight_preprocess_pattern, |
| 179 | + example_inputs=_preprocess_example_inputs, |
| 180 | + is_cuda=is_cuda, |
| 181 | + ) |
| 182 | + |
| 183 | + _remove_pattern_from_graph(graph_module, pattern=scale_match_pattern) |
| 184 | + _remove_pattern_from_graph(graph_module, pattern=weight_preprocess_pattern) |
| 185 | + _remove_denorm_and_late_bias(graph_module) |
| 186 | + |
| 187 | + graph_module.graph.eliminate_dead_code() |
| 188 | + graph_module.recompile() |
| 189 | + |
| 190 | + return PassResult(graph_module, True) |
0 commit comments