Skip to content

Commit b863992

Browse files
authored
Qualcomm AI Engine Direct - Support SLC allocator feature (pytorch#17302)
Summary: - Support SLC allocator feature setting - Support spill-fill buffer for hybrid mode - Fixed redundant convert_op in 8w8a quantization config Test Plan: - Check the debug log to confirm if the SLC allocator is enabled. ``` I [Qnn ExecuTorch]: QnnDsp <V> SLC_ALLOCATOR is set to 1 for graph 0 ``` - Check the debug log whether the spill-fill buffer is configured for the LLM in hybrid mode. ``` python examples/qualcomm/oss_scripts/llama/llama.py -b build-android -H ${HOST} -s ${DEVICE} -m SM8750 --temperature 0 --model_mode hybrid --max_seq_len 128 --prefill_ar_len 32 --decoder_model smollm2_135m --prompt "I would like to learn python, could you teach me with a simple example?" --artifact {ARTIFACTS} ```
1 parent 0350a8b commit b863992

14 files changed

Lines changed: 132 additions & 22 deletions

File tree

‎backends/qualcomm/aot/python/PyQnnManagerAdaptor.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ PYBIND11_MODULE(PyQnnManagerAdaptor, m) {
207207
.def("Init", &PyQnnManager::Init)
208208
.def("InitBackend", &PyQnnManager::InitBackend)
209209
.def("InitContext", &PyQnnManager::InitContext)
210+
.def("InitContextCache", &PyQnnManager::InitContextCache)
210211
.def("IsNodeSupportedByBackend", &PyQnnManager::IsNodeSupportedByBackend)
211212
.def(
212213
"Compile",

‎backends/qualcomm/aot/python/PyQnnManagerAdaptor.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,10 @@ class PyQnnManager {
236236
return qnn_manager_->InitContext(std::optional{graph_names});
237237
}
238238

239+
executorch::runtime::Error InitContextCache() {
240+
return qnn_manager_->InitContextCache();
241+
}
242+
239243
bool IsNodeSupportedByBackend(
240244
std::vector<std::shared_ptr<OpWrapper>>& op_wrappers) {
241245
return qnn_manager_->IsNodeSupportedByBackend(op_wrappers);

‎backends/qualcomm/quantizer/qconfig.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -114,14 +114,21 @@ def get_8a8w_qnn_ptq_config(
114114
# the smallest scale defaults to DEFAULT_EPS_8BIT
115115
extra_args: Dict[str, Any] = {"eps": eps if eps else DEFAULT_EPS_8BIT}
116116

117-
act_quantization_spec = QuantizationSpec(
118-
dtype=torch.uint8,
119-
qscheme=(
120-
torch.per_tensor_symmetric if act_symmetric else torch.per_tensor_affine
121-
),
122-
ch_axis=0,
123-
observer_or_fake_quant_ctr=act_observer.with_args(**extra_args),
124-
)
117+
if act_symmetric:
118+
act_quantization_spec = QuantizationSpec(
119+
dtype=torch.uint8,
120+
qscheme=(torch.per_tensor_symmetric),
121+
ch_axis=0,
122+
observer_or_fake_quant_ctr=act_observer.with_args(**extra_args),
123+
)
124+
else:
125+
act_quantization_spec = QuantizationSpec(
126+
dtype=torch.uint8,
127+
quant_min=torch.iinfo(torch.uint8).min,
128+
quant_max=torch.iinfo(torch.uint8).max,
129+
qscheme=(torch.per_tensor_affine),
130+
observer_or_fake_quant_ctr=act_observer.with_args(**extra_args),
131+
)
125132

126133
weight_quantization_spec = QuantizationSpec(
127134
dtype=torch.int8,

‎backends/qualcomm/runtime/QnnManager.cpp‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,37 @@ Error QnnManager::InitContext(
296296
return Error::Ok;
297297
}
298298

299+
Error QnnManager::InitContextCache() {
300+
if (backend_params_ptr_->backend_init_state_ ==
301+
BackendInitializeState::UNINITIALIZED) {
302+
QNN_EXECUTORCH_LOG_INFO(
303+
"Initialize Qnn backend "
304+
"parameters for Qnn executorch backend type %d",
305+
options_->backend_options()->backend_type());
306+
backend_params_ptr_ = QnnBackendFactory().Create(
307+
backend_bundle_ptr_->implementation.get(),
308+
backend_bundle_ptr_->qnn_backend_ptr.get(),
309+
backend_bundle_ptr_->qnn_device_ptr.get(),
310+
qnn_context_blob_,
311+
options_,
312+
qnn_dlc_manager_.get());
313+
ET_CHECK_OR_RETURN_ERROR(
314+
backend_params_ptr_ != nullptr,
315+
Internal,
316+
"Failed to load Qnn backend.");
317+
// Note: For online_prepare or deserialization, the graph name will be
318+
// obtained from the binary.
319+
ET_CHECK_OR_RETURN_ERROR(
320+
backend_params_ptr_->qnn_backend_cache_ptr_->Configure({}) == Error::Ok,
321+
Internal,
322+
"Fail to configure Qnn backend cache");
323+
324+
backend_params_ptr_->backend_init_state_ =
325+
BackendInitializeState::INITIALIZED;
326+
}
327+
return Error::Ok;
328+
}
329+
299330
Error QnnManager::AllocateTensor(const std::string& graph_name) {
300331
std::vector<Qnn_Tensor_t> input_tensors =
301332
backend_params_ptr_->qnn_context_ptr_->GetGraphInputs(graph_name);

‎backends/qualcomm/runtime/QnnManager.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ class QnnManager {
3838
// graph name will be obtained from the binary.
3939
executorch::runtime::Error InitContext(
4040
std::optional<std::vector<std::string>> graph_names = std::nullopt);
41+
// This function only initialize the context cache to get spill fill buffer
42+
// size
43+
executorch::runtime::Error InitContextCache();
4144
executorch::runtime::Error AllocateTensor(const std::string& graph_name);
4245
executorch::runtime::Error AllocateTensor(
4346
const std::string& graph_name,

‎backends/qualcomm/runtime/backends/QnnBackendFactory.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
5151
"use_dlbc in htp_options: %d", htp_options->use_dlbc());
5252
QNN_EXECUTORCH_LOG_INFO(
5353
"use_fold_relu in htp_options: %d", htp_options->use_fold_relu());
54+
QNN_EXECUTORCH_LOG_INFO(
55+
"use_slc_allocator in htp_options: %d",
56+
htp_options->use_slc_allocator());
5457
QNN_EXECUTORCH_LOG_INFO(
5558
"use_multi_contexts in htp_options: %d",
5659
htp_options->use_multi_contexts());

‎backends/qualcomm/runtime/backends/htp/HtpBackendCache.cpp‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@ Error HtpBackendCache::RetrieveBackendBinaryInfo(
1818
const QnnSystemContext_BinaryInfo_t* binaryinfo) {
1919
QnnHtpSystemContext_HwBlobInfo_t* htp_hwblobinfo = nullptr;
2020
#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21)
21-
QnnHtpSystemContext_GraphBlobInfo_t* htp_graphblobinfo = nullptr;
21+
std::vector<QnnHtpSystemContext_GraphBlobInfo_t*> htp_graphblobinfos;
22+
std::uint32_t num_graphs;
23+
2224
#endif
2325

2426
if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_1) {
@@ -29,8 +31,13 @@ Error HtpBackendCache::RetrieveBackendBinaryInfo(
2931
binaryinfo->contextBinaryInfoV2.hwInfoBlob);
3032
#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21)
3133
} else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3) {
32-
htp_graphblobinfo = static_cast<QnnHtpSystemContext_GraphBlobInfo_t*>(
33-
binaryinfo->contextBinaryInfoV3.graphs->graphInfoV3.graphBlobInfo);
34+
num_graphs = binaryinfo->contextBinaryInfoV3.numGraphs;
35+
for (size_t i = 0; i < num_graphs; ++i) {
36+
htp_graphblobinfos.push_back(
37+
static_cast<QnnHtpSystemContext_GraphBlobInfo_t*>(
38+
binaryinfo->contextBinaryInfoV3.graphs[i]
39+
.graphInfoV3.graphBlobInfo));
40+
}
3441
#endif
3542
} else {
3643
QNN_EXECUTORCH_LOG_WARN(
@@ -51,15 +58,24 @@ Error HtpBackendCache::RetrieveBackendBinaryInfo(
5158
}
5259

5360
#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21)
54-
if (htp_graphblobinfo) {
55-
if (htp_graphblobinfo->version ==
61+
if (htp_graphblobinfos.size() > 0) {
62+
// After version 2.21, we need to get spill fill buffer size from graph
63+
// blob info instead of hw blob info. If there are multiple graphs, we
64+
// should use the max value among all graphs.
65+
if (htp_graphblobinfos[0]->version ==
5666
QNN_SYSTEM_CONTEXT_HTP_GRAPH_INFO_BLOB_VERSION_V1) {
57-
spill_fill_buf_ =
58-
(*htp_graphblobinfo).contextBinaryGraphBlobInfoV1.spillFillBufferSize;
67+
for (size_t i = 0; i < num_graphs; ++i) {
68+
uint64_t spill_fill_buf =
69+
(*htp_graphblobinfos[i])
70+
.contextBinaryGraphBlobInfoV1.spillFillBufferSize;
71+
if (spill_fill_buf > spill_fill_buf_) {
72+
spill_fill_buf_ = spill_fill_buf;
73+
}
74+
}
5975
} else {
6076
QNN_EXECUTORCH_LOG_WARN(
6177
"Unknown QNN Htp graph blob info version %d.",
62-
htp_graphblobinfo->version);
78+
htp_graphblobinfos[0]->version);
6379
return Error::Internal;
6480
}
6581
}

‎backends/qualcomm/runtime/backends/htp/HtpGraphCustomConfig.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,14 @@ HtpGraphCustomConfig::CreateGraphCustomConfigCommon(
7070
htp_options_->use_dlbc() ? 1.0 : 0.0;
7171
ret.push_back(static_cast<QnnGraph_CustomConfig_t>(p_custom_config));
7272

73+
p_custom_config = AllocGraphCustomConfig();
74+
p_custom_config->option = QNN_HTP_GRAPH_CONFIG_OPTION_OPTIMIZATION;
75+
p_custom_config->optimizationOption.type =
76+
QNN_HTP_GRAPH_OPTIMIZATION_TYPE_ENABLE_SLC_ALLOCATOR;
77+
p_custom_config->optimizationOption.floatValue =
78+
htp_options_->use_slc_allocator() ? 1.0 : 0.0;
79+
ret.push_back(static_cast<QnnGraph_CustomConfig_t>(p_custom_config));
80+
7381
return ret;
7482
}
7583
} // namespace qnn

‎backends/qualcomm/serialization/qc_compiler_spec.fbs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,11 @@ table QnnExecuTorchHtpBackendOptions {
181181
/// When multiple graphs appear inside the same context,
182182
/// weights could be reused across all graphs.
183183
use_weight_sharing:bool;
184+
185+
/// Allows user to enable the usage of the System Level Cache Allocator for a given graph.
186+
/// It will help the by reducing overall bandwith on the use case.
187+
/// The feature is only supported by specific SOCs.
188+
use_slc_allocator:bool;
184189
}
185190

186191
/// Logging level of the delegate and QNN backend.

‎backends/qualcomm/serialization/qc_schema.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ class QnnExecuTorchHtpBackendOptions:
166166
use_fold_relu: bool = True
167167
use_multi_contexts: bool = False
168168
use_weight_sharing: bool = False
169+
use_slc_allocator: bool = False
169170

170171

171172
@unique

0 commit comments

Comments
 (0)