Skip to content

Commit 47dc1de

Browse files
chenweng-quicchenweng-quic
authored
Qualcomm AI Engine Direct - Fix static cache (pytorch#16423)
### Summary - Fix static cache used by t5 and whisper - Fix online prepare on Mutable buffer - Fix minor issues ### Test plan ``` python ${EXECUTORCH_ROOT}/backends/qualcomm/tests/test_qnn_delegate.py -k test_t5 --model SM8650 --build_folder build-android --host <host> --device <device_id>--executorch_root ${EXECUTORCH_ROOT} --artifact_dir ./t5_artifact --qa_dataset SQuAD-v1.1.csv python ${EXECUTORCH_ROOT}/backends/qualcomm/tests/test_qnn_delegate.py -k test_whisper --model SM8650 --build_folder build-android --host <host> --device <device_id>--executorch_root ${EXECUTORCH_ROOT} --artifact_dir ./whisper_artifact_dir ``` cc @cccclai @cbilgin
1 parent 913436a commit 47dc1de

13 files changed

Lines changed: 207 additions & 57 deletions

File tree

‎backends/qualcomm/_passes/convert_bmm_to_matmul.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class ConvertBmmToMatmul(ExportPass):
2525
bmm = exir_ops.edge.aten.bmm.default
2626
matmul = exir_ops.edge.aten.matmul.default
2727
patterns = [
28+
{view_copy: 3, bmm: 1},
2829
{expand_copy: 2, view_copy: 3, bmm: 1},
2930
{expand_copy: 2, view_copy: 3, bmm: 1, clone: 1},
3031
{bmm: 1},

‎backends/qualcomm/_passes/remove_redundancy.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ def __init__(self, quantization_capture=False):
2323
torch.ops.aten.alias.default: self._default_condition,
2424
exir_ops.edge.aten.alias.default: self._default_condition,
2525
exir_ops.edge.aten.alias_copy.default: self._default_condition,
26+
exir_ops.edge.aten.expand_copy.default: self._same_shape_condition,
2627
exir_ops.edge.aten.lift_fresh_copy.default: self._default_condition,
28+
exir_ops.edge.aten.repeat.default: self._same_shape_condition,
2729
# remove this target if '_skip_dim_order' is set to False
2830
exir_ops.edge.dim_order_ops._to_dim_order_copy.default: self._dim_order_op_condition,
2931
# remove channel_last / contiguous _to_copy if '_skip_dim_order' is set to True
@@ -43,6 +45,9 @@ def __init__(self, quantization_capture=False):
4345
def _dim_order_op_condition(self, node):
4446
return node.meta["val"].dtype == node.args[0].meta["val"].dtype
4547

48+
def _same_shape_condition(self, node):
49+
return node.args[0].meta["val"].shape == node.meta["val"].shape
50+
4651
def _to_copy_op_condition(self, node):
4752
return "memory_format" in node.kwargs
4853

‎backends/qualcomm/builders/op_copy.py‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@
77

88
import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManager
99

10+
import numpy as np
1011
import torch
1112
from executorch.backends.qualcomm.utils.constants import QCOM_QUANT_ATTRS
1213

1314
from .node_visitor import NodeVisitor
1415
from .node_visitor_manager import register_node_visitor
15-
from .qnn_constants import OpReshape, QNN_OP_PACKAGE_NAME_QTI_AISW
16+
from .qnn_constants import OpReshape, OpTile, QNN_OP_PACKAGE_NAME_QTI_AISW
1617

1718

1819
@register_node_visitor
@@ -27,38 +28,85 @@ def define_node(
2728
node: torch.fx.Node,
2829
nodes_to_wrappers: Dict[torch.fx.Node, PyQnnManager.TensorWrapper],
2930
) -> PyQnnManager.PyQnnOpWrapper:
31+
# The aten copy support broadcasting, and therefore translated to
32+
# Reshape and Tile.
33+
# e.g., torch.ops.aten.copy.default(torch.rand(3,4,5), torch.rand(4,1))
3034
input_node = self.get_node(node.args[1])
3135
input_tensor = self.get_tensor(input_node, node)
32-
copy_inp_tensor_wrapper = self.define_tensor(
36+
output_tensor = self.get_tensor(node, node)
37+
should_insert_tile = input_tensor.numel() != output_tensor.numel()
38+
reshape_node_name = (
39+
node.name + "_unsqueeze" if should_insert_tile else node.name
40+
)
41+
reshape_inp_tensor_wrapper = self.define_tensor(
3342
input_node,
3443
node,
3544
input_tensor,
3645
PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
3746
nodes_to_wrappers,
3847
)
39-
40-
copy_input_tensors = [copy_inp_tensor_wrapper]
41-
48+
reshape_input_tensors = [reshape_inp_tensor_wrapper]
4249
if quant_attrs := input_node.meta.get(QCOM_QUANT_ATTRS):
4350
quant_attrs = quant_attrs.copy()
4451
# Because there is no output after convert_pt2e, the QCOM_QUANT_ATTRS of node is none
4552
node.meta[QCOM_QUANT_ATTRS] = quant_attrs
46-
output_tensor = self.get_tensor(node, node)
47-
output_tensor_wrapper = self.define_tensor(
48-
node,
49-
node,
50-
output_tensor,
51-
PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
52-
nodes_to_wrappers,
53+
54+
reshape_tensor = input_tensor
55+
while len(reshape_tensor.shape) < len(output_tensor.shape):
56+
reshape_tensor = reshape_tensor.unsqueeze(0)
57+
input_quant_encoding, input_quant_configs = self.get_quant_encoding_conf(
58+
input_node, node
59+
)
60+
reshape_tensor_wrapper = self.define_custom_tensor_wrapper(
61+
node_name=reshape_node_name,
62+
tensor_type=PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
63+
dtype=self.get_data_type(reshape_tensor, input_quant_configs),
64+
quant_encoding=input_quant_encoding,
65+
quant_configs=input_quant_configs,
66+
dims=reshape_tensor.size(),
67+
tensor=reshape_tensor,
68+
is_fake_tensor=True,
69+
nodes_to_wrappers=nodes_to_wrappers,
5370
)
54-
copy_output_tensors = [output_tensor_wrapper]
71+
reshape_output_tensors = [reshape_tensor_wrapper]
5572

56-
copy_op = PyQnnManager.PyQnnOpWrapper(
57-
node.name,
73+
reshape_op = PyQnnManager.PyQnnOpWrapper(
74+
reshape_node_name,
5875
QNN_OP_PACKAGE_NAME_QTI_AISW,
5976
OpReshape.op_name,
6077
)
61-
copy_op.AddInputTensors(copy_input_tensors)
62-
copy_op.AddOutputTensors(copy_output_tensors)
63-
64-
return copy_op
78+
reshape_op.AddInputTensors(reshape_input_tensors)
79+
reshape_op.AddOutputTensors(reshape_output_tensors)
80+
op_wrapper_list = [reshape_op]
81+
if should_insert_tile:
82+
output_tensor_wrapper = self.define_tensor(
83+
node,
84+
node,
85+
output_tensor,
86+
PyQnnManager.Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE,
87+
nodes_to_wrappers,
88+
)
89+
tile_output_tensors = [output_tensor_wrapper]
90+
tile_op = PyQnnManager.PyQnnOpWrapper(
91+
node.name,
92+
QNN_OP_PACKAGE_NAME_QTI_AISW,
93+
OpTile.op_name,
94+
)
95+
multiples = []
96+
for i in range(len(reshape_tensor.shape)):
97+
assert (
98+
output_tensor.shape[i] % reshape_tensor.shape[i] == 0
99+
), f"Shape mismatch at dim {i}: {output_tensor.shape[i]} not divisible by {reshape_tensor.shape[i]}"
100+
multiples.append(output_tensor.shape[i] // reshape_tensor.shape[i])
101+
tile_op.AddTensorParam(
102+
OpTile.param_multiples,
103+
PyQnnManager.Qnn_DataType_t.QNN_DATATYPE_UINT_32,
104+
1,
105+
[len(reshape_tensor.shape)],
106+
np.array(multiples, dtype=np.uint32),
107+
True,
108+
)
109+
tile_op.AddInputTensors(reshape_output_tensors)
110+
tile_op.AddOutputTensors(tile_output_tensors)
111+
op_wrapper_list.append(tile_op)
112+
return op_wrapper_list

‎backends/qualcomm/builders/op_index_put.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ def define_node( # noqa: C901
198198
nodes_to_wrappers=nodes_to_wrappers,
199199
)
200200
tile_op = PyQnnManager.PyQnnOpWrapper(
201-
node.name,
201+
node.name + f"_indices_tile_{i}",
202202
QNN_OP_PACKAGE_NAME_QTI_AISW,
203203
OpTile.op_name,
204204
)
@@ -231,7 +231,7 @@ def define_node( # noqa: C901
231231
nodes_to_wrappers=nodes_to_wrappers,
232232
)
233233
reshape_op = PyQnnManager.PyQnnOpWrapper(
234-
node.name,
234+
node.name + f"_reshape_{i}",
235235
QNN_OP_PACKAGE_NAME_QTI_AISW,
236236
OpReshape.op_name,
237237
)
@@ -265,7 +265,7 @@ def define_node( # noqa: C901
265265
nodes_to_wrappers=nodes_to_wrappers,
266266
)
267267
tile_op = PyQnnManager.PyQnnOpWrapper(
268-
node.name,
268+
node.name + f"_tile_{i}",
269269
QNN_OP_PACKAGE_NAME_QTI_AISW,
270270
OpTile.op_name,
271271
)
@@ -309,7 +309,7 @@ def define_node( # noqa: C901
309309
nodes_to_wrappers=nodes_to_wrappers,
310310
)
311311
concat_op = PyQnnManager.PyQnnOpWrapper(
312-
node.name,
312+
node.name + "_concat",
313313
QNN_OP_PACKAGE_NAME_QTI_AISW,
314314
OpConcat.op_name,
315315
)
@@ -367,7 +367,7 @@ def define_node( # noqa: C901
367367
nodes_to_wrappers=nodes_to_wrappers,
368368
)
369369
value_reshape_op = PyQnnManager.PyQnnOpWrapper(
370-
node.name,
370+
node.name + "_value_reshape",
371371
QNN_OP_PACKAGE_NAME_QTI_AISW,
372372
OpReshape.op_name,
373373
)
@@ -404,7 +404,7 @@ def define_node( # noqa: C901
404404
nodes_to_wrappers=nodes_to_wrappers,
405405
)
406406
value_tile_op = PyQnnManager.PyQnnOpWrapper(
407-
node.name,
407+
node.name + "_value_tile",
408408
QNN_OP_PACKAGE_NAME_QTI_AISW,
409409
OpTile.op_name,
410410
)
@@ -461,7 +461,7 @@ def define_node( # noqa: C901
461461
nodes_to_wrappers=nodes_to_wrappers,
462462
)
463463
target_index_reshape_op = PyQnnManager.PyQnnOpWrapper(
464-
node.name,
464+
node.name + "_target_index_reshape",
465465
QNN_OP_PACKAGE_NAME_QTI_AISW,
466466
OpReshape.op_name,
467467
)

‎backends/qualcomm/quantizer/qconfig.py‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
QuantizationSpec,
2929
)
3030

31+
DEFAULT_EPS_8BIT = 0.0001 / 255
32+
DEFAULT_EPS_16BIT = 0.0001 / 65535
33+
3134

3235
@dataclass(eq=True)
3336
class QuantizationConfig:
@@ -104,10 +107,12 @@ def _derive_bias_qparams_fn(
104107

105108

106109
def get_8a8w_qnn_ptq_config(
107-
act_symmetric: bool = False, act_observer=MovingAverageMinMaxObserver
110+
act_symmetric: bool = False,
111+
act_observer=MovingAverageMinMaxObserver,
112+
eps: float = None,
108113
) -> QuantizationConfig:
109-
# the smallest scale: 0.0001 / 255
110-
extra_args: Dict[str, Any] = {"eps": 2**-21}
114+
# the smallest scale defaults to DEFAULT_EPS_8BIT
115+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_8BIT}
111116

112117
act_quantization_spec = QuantizationSpec(
113118
dtype=torch.uint8,
@@ -146,10 +151,12 @@ def get_8a8w_qnn_ptq_config(
146151

147152

148153
def get_8a4w_qnn_ptq_config(
149-
act_symmetric: bool = True, act_observer=MovingAverageMinMaxObserver
154+
act_symmetric: bool = True,
155+
act_observer=MovingAverageMinMaxObserver,
156+
eps: float = None,
150157
) -> QuantizationConfig:
151-
# the smallest scale: 0.0001 / 255
152-
extra_args: Dict[str, Any] = {"eps": 2**-21}
158+
# the smallest defaults to DEFAULT_EPS_8BIT
159+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_8BIT}
153160

154161
if act_symmetric:
155162
# If zero_point is 128, htp can do optimizations.
@@ -203,10 +210,10 @@ def get_8a4w_qnn_ptq_config(
203210

204211
# 4 bits quantization only supports specific ops.
205212
def get_16a4w_qnn_ptq_config(
206-
act_observer=MovingAverageMinMaxObserver,
213+
act_observer=MovingAverageMinMaxObserver, eps: float = None
207214
) -> QuantizationConfig:
208-
# the smallest scale: 0.0001 / 65535
209-
extra_args: Dict[str, Any] = {"eps": 2**-29}
215+
# the smallest defaults to DEFAULT_EPS_16BIT
216+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
210217
act_quantization_spec = QuantizationSpec(
211218
dtype=torch.int32,
212219
quant_min=torch.iinfo(torch.uint16).min,
@@ -243,10 +250,10 @@ def get_16a4w_qnn_ptq_config(
243250

244251

245252
def get_16a8w_qnn_ptq_config(
246-
act_observer=MovingAverageMinMaxObserver,
253+
act_observer=MovingAverageMinMaxObserver, eps: float = None
247254
) -> QuantizationConfig:
248-
# the smallest scale: 0.0001 / 65535
249-
extra_args: Dict[str, Any] = {"eps": 2**-29}
255+
# the smallest defaults to DEFAULT_EPS_16BIT
256+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
250257
act_quantization_spec = QuantizationSpec(
251258
dtype=torch.int32,
252259
quant_min=torch.iinfo(torch.uint16).min,
@@ -281,10 +288,10 @@ def get_16a8w_qnn_ptq_config(
281288

282289

283290
def get_16a8w_qnn_qat_config(
284-
act_observer=MovingAverageMinMaxObserver,
291+
act_observer=MovingAverageMinMaxObserver, eps: float = None
285292
) -> QuantizationConfig:
286-
# the smallest scale: 0.0001 / 65535
287-
extra_args: Dict[str, Any] = {"eps": 2**-29}
293+
# the smallest defaults to DEFAULT_EPS_16BIT
294+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
288295
act_fake_quant_ctr = FusedMovingAvgObsFakeQuantize.with_args(
289296
dtype=torch.int32,
290297
quant_min=torch.iinfo(torch.uint16).min,
@@ -339,10 +346,10 @@ def get_16a8w_qnn_qat_config(
339346

340347

341348
def get_16a16w_qnn_ptq_config(
342-
act_observer=MovingAverageMinMaxObserver,
349+
act_observer=MovingAverageMinMaxObserver, eps: float = None
343350
) -> QuantizationConfig:
344-
# the smallest scale: 0.0001 / 65535
345-
extra_args: Dict[str, Any] = {"eps": 2**-29}
351+
# the smallest defaults to DEFAULT_EPS_16BIT
352+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
346353
act_quantization_spec = QuantizationSpec(
347354
dtype=torch.int32,
348355
quant_min=torch.iinfo(torch.uint16).min,
@@ -385,10 +392,10 @@ def get_ptq_per_channel_quant_config(
385392
act_observer=MovingAverageMinMaxObserver,
386393
act_symmetric: bool = False,
387394
ch_axis: int = 0,
395+
eps: float = None,
388396
) -> QuantizationConfig:
389-
# the smallest scale: 0.0001 / 65535
390-
extra_args: Dict[str, Any] = {"eps": 2**-29}
391-
397+
# the smallest defaults to DEFAULT_EPS_16BIT
398+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
392399
supported_act_types = {
393400
torch.uint8,
394401
torch.uint16,
@@ -457,9 +464,10 @@ def get_ptq_per_block_quant_config(
457464
act_observer=MovingAverageMinMaxObserver,
458465
act_symmetric: bool = False,
459466
ch_axis: int = 0,
467+
eps: float = None,
460468
) -> QuantizationConfig:
461-
# the smallest scale: 0.0001 / 65535
462-
extra_args: Dict[str, Any] = {"eps": 2**-29}
469+
# the smallest defaults to DEFAULT_EPS_16BIT
470+
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_16BIT}
463471
quantization_config = get_ptq_per_channel_quant_config(
464472
act_dtype=act_dtype,
465473
weight_dtype=weight_dtype,

‎backends/qualcomm/quantizer/quantizer.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ class ModuleQConfig:
160160
is_conv_per_channel: bool = False
161161
is_linear_per_channel: bool = False
162162
act_observer: Optional[UniformQuantizationObserverBase] = None
163+
eps: Optional[float] = None
163164

164165
def __post_init__(self):
165166
if (self.quant_dtype, self.is_qat) not in QUANT_CONFIG_DICT:
@@ -172,9 +173,9 @@ def __post_init__(self):
172173
per_block_quant_config_func,
173174
) = QUANT_CONFIG_DICT[(self.quant_dtype, self.is_qat)]
174175
self.quant_config = (
175-
quant_config_func(act_observer=self.act_observer)
176+
quant_config_func(act_observer=self.act_observer, eps=self.eps)
176177
if self.act_observer
177-
else quant_config_func()
178+
else quant_config_func(eps=self.eps)
178179
)
179180

180181
# Assume per_channel_quant/per_block_quant only happen on axis_0 or axis_1, increase the range if there's a need
@@ -185,10 +186,12 @@ def __post_init__(self):
185186
self.per_channel_quant_config_list.append(
186187
(
187188
per_channel_quant_config_func(
188-
act_observer=self.act_observer, ch_axis=i
189+
act_observer=self.act_observer,
190+
ch_axis=i,
191+
eps=self.eps,
189192
)
190193
if self.act_observer
191-
else per_channel_quant_config_func(ch_axis=i)
194+
else per_channel_quant_config_func(ch_axis=i, eps=self.eps)
192195
)
193196
)
194197

@@ -409,6 +412,7 @@ def set_default_quant_config(
409412
is_conv_per_channel=False,
410413
is_linear_per_channel=False,
411414
act_observer=None,
415+
eps=None,
412416
) -> None:
413417
"""
414418
Set the default quant config for quantizer.
@@ -419,6 +423,7 @@ def set_default_quant_config(
419423
is_conv_per_channel (bool, optional): Enables per-channel quantization for convolution operations.
420424
is_linear_per_channel (bool, optional): Enables per-channel quantization for linear (fully connected) operations.
421425
act_observer (Optional[UniformQuantizationObserverBase], optional): Custom observer for activation quantization. If not specified, the default observer is determined by `QUANT_CONFIG_DICT`.
426+
eps (float): Minimum scale for quantization.
422427
423428
"""
424429
self.default_quant_config = ModuleQConfig(
@@ -427,6 +432,7 @@ def set_default_quant_config(
427432
is_conv_per_channel=is_conv_per_channel,
428433
is_linear_per_channel=is_linear_per_channel,
429434
act_observer=act_observer,
435+
eps=eps,
430436
)
431437

432438
def set_block_size_map(self, block_size_map: Dict[str, Tuple]) -> None:

0 commit comments

Comments
 (0)