Skip to content

Commit 46f9696

Browse files
authored
Qualcomm AI Engine Direct - Pass migration - part 1 (pytorch#20738)
## Summary This is **part 1** of a series whose goal is to deprecate `to_edge_transform_and_lower_to_qnn` and migrate Qualcomm-specific passes from the export pipeline into `to_edge_transform_and_lower`. The core change is in `exir/program/_program.py`: after the `EdgeProgramManager` finishes running `transform_passes`, we now invoke `lift_constant_tensor_pass` on each method's `ExportedProgram`. Several QNN passes (`CanonicalizeConv`, `ConvertLinearToConv2d`, etc.) rewrite constant values and insert new `get_attr` nodes to store them. Without lifting, these mutated constants are not reflected back into the `ExportedProgram`; `lift_constant_tensor_pass` promotes the `get_attr` nodes into placeholders so the modified constants are correctly propagated. ## Changes - `exir/program/_program.py`: run `lift_constant_tensor_pass` over every method after `edge_manager.transform(transform_passes)` completes. - Move `CanonicalizeConv` and `ConvertLinearToConv2d` out of the export pipeline (`get_export_passes`) and into the to-edge transform pipeline. - Add new pass `AnnotateGetAttr` to repopulate `QCOM_QUANT_ATTRS` for `get_attr` nodes. Passes such as `I64toI32` and `LayoutTransform` reconstruct the `GraphModule` (`graph_module = super().call(graph_module).graph_module`), which drops the quant attributes previously stored on `get_attr` node metadata. ## Test plan - `CanonicalizeConv` Test cases: `TEST_MATRIX = {TestQNNFloatingPointOperator.test_qnn_backend_conv1d TestQNNFloatingPointOperator.test_qnn_conv1d_batch_norm TestQNNFloatingPointOperator.test_qnn_backend_conv2d TestQNNFloatingPointOperator.test_qnn_backend_conv3d_sequential TestQNNFloatingPointOperator.test_qnn_backend_conv_transpose1d TestQNNFloatingPointOperator.test_qnn_backend_conv_transpose2d TestQNNFloatingPointOperator.test_qnn_backend_conv_transpose3d TestQNNFloatingPointModel.test_qnn_backend_conv1d_relu_log_softmax TestQNNQuantizedOperator.test_qnn_backend_conv1d TestQNNQuantizedOperator.test_qnn_conv1d_batch_norm TestQNNQuantizedOperator.test_qnn_backend_conv2d TestQNNQuantizedOperator.test_qnn_backend_conv3d_sequential TestQNNQuantizedOperator.test_qnn_backend_conv_transpose1d TestQNNQuantizedOperator.test_qnn_backend_conv_transpose2d TestQNNQuantizedOperator.test_qnn_backend_conv_transpose3d TestQNNQuantizedModel.test_qnn_backend_conv1d_relu_log_softmax}` Command: `python backends/qualcomm/tests/test_qnn_delegate.py ${TEST_MATRIX} --build_folder build-android/ --host ${HOST_NAME} --device ${DEVICE_ID} --soc_model ${SOC_ID} --seed 1126 --backend htp` `python -m pytest "backends/qualcomm/tests/rework/passes/test.py::test_canonicalize_conv[htp]" -v` `python -m pytest "backends/qualcomm/tests/rework/passes/test.py::test_canonicalize_conv[lpai]" -v` - `ConvertLinearToConv2d` Test cases: `TEST_MATRIX = {TestQNNFloatingPointOperator.test_qnn_backend_linear TestQNNFloatingPointOperator.test_qnn_backend_linear_to_conv2d TestQNNFloatingPointOperator.test_qnn_backend_linear_shared_weights TestQNNFloatingPointOperator.test_qnn_backend_linear_to_conv2d_shared_weights TestQNNQuantizedOperator.test_qnn_backend_linear TestQNNQuantizedOperator.test_qnn_backend_linear_to_conv2d TestQNNQuantizedOperator.test_qnn_backend_linear_shared_weights TestQNNQuantizedOperator.test_qnn_backend_linear_to_conv2d_shared_weights TestQNNQuantizedOperator.test_qnn_backend_linear_block TestQNNQuantizedOperator.test_qnn_backend_linear_to_conv2d_block TestQNNQuantizedOperator.test_qnn_backend_linear_qat}` Command: `python backends/qualcomm/tests/test_qnn_delegate.py ${TEST_MATRIX} --build_folder build-android/ --host ${HOST_NAME} --device ${DEVICE_ID} --soc_model ${SOC_ID} --seed 1126 --backend htp` `python -m pytest "backends/qualcomm/tests/rework/passes/test.py::test_convert_linear_to_conv2d[htp]" -v` `python -m pytest "backends/qualcomm/tests/rework/passes/test.py::test_convert_linear_to_conv2d[lpai]" -v` `python -m pytest "backends/qualcomm/tests/rework/htp/op/v68/test.py::test_linear_shared_weights" -v` - `AnnotateGetAttr` Command: `python -m pytest "backends/qualcomm/tests/rework/passes/test.py::test_annotate_get_attr" -v`
1 parent 7454b89 commit 46f9696

21 files changed

Lines changed: 957 additions & 474 deletions

‎backends/qualcomm/_passes/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# LICENSE file in the root directory of this source tree.
66

77
from .annotate_avg_pool1d import AnnotateAvgPool1D
8+
from .annotate_get_attr import AnnotateGetAttr
89
from .annotate_quant_attrs import AnnotateQuantAttrs
910
from .annotate_stack import AnnotateStack
1011
from .annotate_unbind import AnnotateUnbind
@@ -73,6 +74,7 @@
7374

7475
__all__ = [
7576
AnnotateAvgPool1D,
77+
AnnotateGetAttr,
7678
AnnotateQuantAttrs,
7779
AnnotateStack,
7880
AnnotateUnbind,
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Copyright (c) Qualcomm Innovation Center, Inc.
2+
# All rights reserved
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
from executorch.backends.qualcomm.builders.node_visitor import dq_ops
9+
from executorch.backends.qualcomm.utils.constants import QCOM_QUANT_ATTRS
10+
from executorch.exir.pass_base import ExportPass, PassResult
11+
12+
from .utils import get_quant_attrs
13+
14+
15+
class AnnotateGetAttr(ExportPass):
16+
"""
17+
Annotates quantization attributes for `get_attr` nodes.
18+
19+
In passes such as `CanonicalizeConv`, `ConvertLinearToConv2d`,
20+
`LiftConstantScalarOperands`, and others, `get_attr` nodes and the
21+
corresponding quantization attributes are inserted into the GraphModule
22+
to store modified constant values.
23+
24+
However, the quantization attributes associated with `get_attr` nodes will
25+
be discarded in certain passes, such as `I64toI32` and `LayoutTransform`.
26+
This happens due to the following line:
27+
`graph_module = super().call(graph_module).graph_module`
28+
29+
which reconstructs the GraphModule and drops any existing quantization
30+
attributes stored in the metadata of `get_attr` nodes.
31+
32+
To guarantee correctness, this pass repopulates the quantization attributes
33+
for `get_attr` nodes and ensures to be scheduled after the `I64toI32` and
34+
`LayoutTransform` passes.
35+
"""
36+
37+
def __init__(self, edge_program: torch.export.ExportedProgram):
38+
self.edge_program = edge_program
39+
40+
def _annotate_get_attr(
41+
self, graph_module: torch.fx.GraphModule
42+
) -> torch.fx.GraphModule:
43+
for node in graph_module.graph.nodes:
44+
if node.op == "get_attr":
45+
for op in list(node.users):
46+
if op.target in dq_ops:
47+
quant_attrs = get_quant_attrs(self.edge_program, op)
48+
node.meta[QCOM_QUANT_ATTRS] = quant_attrs
49+
break
50+
51+
def call(self, graph_module: torch.fx.GraphModule):
52+
self._annotate_get_attr(graph_module)
53+
graph_module.recompile()
54+
return PassResult(graph_module, True)

‎backends/qualcomm/_passes/annotate_quant_attrs.py‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,6 @@ def _annotate_source_nodes(
5555
source_n = quant_node.args[0]
5656
source_n.meta[QCOM_QUANT_ATTRS] = quant_attrs
5757

58-
def _expand(self, tensor, dim, axis) -> torch.Tensor:
59-
tensor = tensor[(...,) + (None,) * (dim - 1)]
60-
order = torch.arange(dim).tolist()
61-
order[axis], order[0] = order[0], order[axis]
62-
return tensor.permute(order)
63-
6458
# Find the the last dq nodes between regular op nodes
6559
# Return dq2 in example below when q1 is given as node parameter:
6660
# ... -> n1 -> q1 -> dq1 -> q2 -> dq2 -> n2 -> ...

‎backends/qualcomm/_passes/backends/gpu/qnn_gpu_pass_manager.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,9 @@ def get_annotation_passes(cls):
3838
return []
3939

4040
@classmethod
41-
def get_export_passes(
42-
cls,
43-
convert_linear_to_conv2d: bool = False,
44-
):
41+
def get_export_passes(cls):
4542
# DecomposeReciprocal should be placed in the export pipeline, as it depends on
4643
# LiftConstantScalarOperands to lift the scalar operand.
4744
passes = [DecomposeReciprocal]
48-
passes.extend(super().get_export_passes(convert_linear_to_conv2d))
45+
passes.extend(super().get_export_passes())
4946
return passes

‎backends/qualcomm/_passes/backends/htp/qnn_htp_pass_manager.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,9 @@ def get_annotation_passes(cls):
4242
return passes
4343

4444
@classmethod
45-
def get_export_passes(
46-
cls,
47-
convert_linear_to_conv2d: bool = False,
48-
):
45+
def get_export_passes(cls):
4946
# DecomposeReciprocal should be placed in the export pipeline, as it depends on
5047
# LiftConstantScalarOperands to lift the scalar operand.
5148
passes = [DecomposeReciprocal]
52-
passes.extend(super().get_export_passes(convert_linear_to_conv2d))
49+
passes.extend(super().get_export_passes())
5350
return passes

‎backends/qualcomm/_passes/backends/lpai/qnn_lpai_pass_manager.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,14 +70,11 @@ def get_annotation_passes(cls):
7070
return passes
7171

7272
@classmethod
73-
def get_export_passes(
74-
cls,
75-
convert_linear_to_conv2d: bool = False,
76-
):
73+
def get_export_passes(cls):
7774
# Both DecomposeHardSigmoid and DecomposeReciprocal should be placed in the export
7875
# pipeline, as they rely on LiftConstantScalarOperands to lift the scalar operand.
7976
passes = [DecomposeHardsigmoid, DecomposeReciprocal]
80-
passes.extend(super().get_export_passes(convert_linear_to_conv2d))
77+
passes.extend(super().get_export_passes())
8178
return passes
8279

8380
@classmethod

0 commit comments

Comments
 (0)