Skip to content

Commit 575f863

Browse files
authored
Qualcomm AI Engine Direct - key manager caches by backend and SoC (pytorch#22933)
## Summary FCB lowering can create multiple QNN backend managers within one process, each targeting a different SoC. Previously, the manager and backend-bundle caches used backend type alone as their key, allowing a manager or device configuration created for one target SoC to be reused incorrectly for another. This PR makes those caches FCB-safe by keying them with `(backend type, SoC)`. Each target receives its own manager and backend bundle, while repeated requests for the same target continue to reuse the cached instance. It also consolidates QNN API compatibility checks in `QnnSdkCompatibility.h` and applies the shared helper to HTP cache handling, IR support, and LPAI configuration. ## Compatibility `PyQnnManager.Compile()` now returns `bytes` rather than a NumPy `uint8` array. This is an intentional internal binding API alignment: it avoids the in-tree NumPy-to-bytes conversion, but downstream callers relying on NumPy-array behavior must adapt. The binding still copies from the QNN context buffer; this is not a zero-copy optimization. ## Validation - `lintrunner --skip MYPY -a` on the changed files - Incremental `build-x86` QNN build - `pytest -q backends/qualcomm/tests/test_qnn_manager_device_cache.py backends/qualcomm/tests/test_import_side_effects.py` (60 passed) ## AI assistance This PR was authored with OpenAI Codex.
1 parent e9e21b5 commit 575f863

18 files changed

Lines changed: 150 additions & 66 deletions

‎backends/qualcomm/aot/python/PyQnnManagerAdaptor.h‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ class PyQnnManager {
245245
return qnn_manager_->IsNodeSupportedByBackend(op_wrappers);
246246
}
247247

248-
py::array_t<char> Compile(
248+
py::bytes Compile(
249249
const std::vector<std::string>& graph_names,
250250
std::vector<std::vector<std::shared_ptr<OpWrapper>>>& op_wrappers) {
251251
QnnExecuTorchContextBinary binary_info;
@@ -254,23 +254,18 @@ class PyQnnManager {
254254
if (qnn_manager_->Compile(graph_names[i], op_wrappers[i]) !=
255255
executorch::runtime::Error::Ok) {
256256
QNN_EXECUTORCH_LOG_ERROR("Fail to compile QNN graph");
257-
return py::array_t<char>(0);
257+
return py::bytes("", 0);
258258
}
259259
}
260260
auto qnn_executorch_options = GetQnnExecuTorchOptions(
261261
qnn_executorch_option_ptr_.cast<std::string_view>().data());
262262
if (qnn_executorch_options->saver() ||
263263
qnn_manager_->GetContextBinary(binary_info) !=
264264
executorch::runtime::Error::Ok) {
265-
return py::array_t<char>(0);
265+
return py::bytes("", 0);
266266
}
267-
268-
// allocate py::array (to pass the result of the C++ function to Python)
269-
auto result = py::array_t<char>(binary_info.nbytes);
270-
auto result_buffer = result.request();
271-
char* result_ptr = (char*)result_buffer.ptr;
272-
std::memcpy(result_ptr, binary_info.buffer, binary_info.nbytes);
273-
return result;
267+
return py::bytes(
268+
reinterpret_cast<const char*>(binary_info.buffer), binary_info.nbytes);
274269
}
275270

276271
void Destroy() {

‎backends/qualcomm/qnn_preprocess.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ def _build_op_wrappers(
138138
op = unwrap_op_overload(node.target)
139139
context_loader_target = eval(
140140
f"torch.ops.{OpContextLoader.namespace}.{op.__name__}",
141-
globals().update(torch.__dict__),
141+
{"torch": torch},
142142
)
143143
assert op == context_loader_target, err_msg
144144
# if graph has context binary loader node, return directly

‎backends/qualcomm/runtime/backends/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ target_sources(
1515
target_sources(
1616
qnn_function_interface
1717
INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnFunctionInterface.h
18+
INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnSdkCompatibility.h
1819
)
1920

2021
# qnn_sys_implementation
@@ -28,6 +29,7 @@ target_sources(
2829
target_sources(
2930
qnn_sys_function_interface
3031
INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnSysFunctionInterface.h
32+
INTERFACE ${CMAKE_CURRENT_LIST_DIR}/QnnSdkCompatibility.h
3133
)
3234

3335
# qnn_logger

‎backends/qualcomm/runtime/backends/QnnBackendCache.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
#include <executorch/backends/qualcomm/runtime/backends/QnnBackendCache.h>
1010
#include <executorch/backends/qualcomm/runtime/backends/QnnCustomProtocol.h>
11+
#include <executorch/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h>
1112

1213
namespace executorch {
1314
namespace backends {
@@ -50,7 +51,7 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary(
5051
} else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_2) {
5152
num_graphs = binaryinfo->contextBinaryInfoV2.numGraphs;
5253
graphs = binaryinfo->contextBinaryInfoV2.graphs;
53-
#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21)
54+
#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21)
5455
} else if (binaryinfo->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3) {
5556
num_graphs = binaryinfo->contextBinaryInfoV3.numGraphs;
5657
graphs = binaryinfo->contextBinaryInfoV3.graphs;
@@ -66,7 +67,7 @@ Error QnnBackendCache::GetQnnGraphInfoFromBinary(
6667
RetrieveGraphInfo<QnnSystemContext_GraphInfoV1_t>(graphs[i].graphInfoV1);
6768
} else if (graphs->version == QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_2) {
6869
RetrieveGraphInfo<QnnSystemContext_GraphInfoV2_t>(graphs[i].graphInfoV2);
69-
#if (QNN_API_VERSION_MAJOR >= 2 && QNN_API_VERSION_MINOR >= 21)
70+
#if QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(2, 21)
7071
} else if (graphs->version == QNN_SYSTEM_CONTEXT_GRAPH_INFO_VERSION_3) {
7172
RetrieveGraphInfo<QnnSystemContext_GraphInfoV3_t>(graphs[i].graphInfoV3);
7273
#endif

‎backends/qualcomm/runtime/backends/QnnBackendCommon.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,12 @@
1212
#include <executorch/backends/qualcomm/runtime/backends/QnnImplementation.h>
1313
#include <executorch/backends/qualcomm/runtime/backends/QnnLogger.h>
1414
#include <executorch/backends/qualcomm/runtime/backends/QnnOpPackageManager.h>
15+
#include <executorch/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h>
1516
#include <unordered_set>
1617
#include <vector>
1718

1819
#include "HTP/QnnHtpCommon.h"
1920
#include "QnnBackend.h"
20-
#include "QnnCommon.h"
2121
#include "QnnTypes.h"
2222
#include "Saver/QnnSaverCommon.h"
2323

‎backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle(
5050
get_option(options->log_level(), QNN_RUNTIME_LOG_LEVEL);
5151
QnnExecuTorchBackendType backend_type =
5252
options->backend_options()->backend_type();
53+
const auto bundle_key =
54+
std::make_pair(backend_type, options->soc_info()->soc_model());
5355

5456
if (current_lib_path.empty()) {
5557
switch (backend_type) {
@@ -76,7 +78,7 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle(
7678
}
7779

7880
// Check if resources already exist
79-
auto it = qnn_backend_bundles_map_.find(backend_type);
81+
auto it = qnn_backend_bundles_map_.find(bundle_key);
8082
if (it != qnn_backend_bundles_map_.end()) {
8183
// Create new shared_ptr that shares ownership of the managed object.
8284
if (auto existing_bundle = it->second.lock()) {
@@ -171,8 +173,7 @@ Error QnnBackendUnifiedRegistry::GetOrCreateBackendBundle(
171173
// weak_ptr under this key, and emplace will not replace it, so every later
172174
// request would miss the cache and build another backend for the same type.
173175
// CleanupExpired() cannot be used from here -- it takes mutex_, already held.
174-
qnn_backend_bundles_map_.insert_or_assign(
175-
backend_type, bundle); // Store weak_ptr to the bundle
176+
qnn_backend_bundles_map_.insert_or_assign(bundle_key, bundle);
176177

177178
return Error::Ok;
178179
}

‎backends/qualcomm/runtime/backends/QnnBackendUnifiedRegistry.h‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include <pal/Path.h>
1919

20+
#include <map>
2021
#include <memory>
2122
#include <mutex>
2223
#include <string>
@@ -109,9 +110,9 @@ class QnnBackendUnifiedRegistry {
109110
}
110111
}
111112

112-
// Stores the collection of shared resources, with backend_type being used as
113-
// the key.
114-
std::unordered_map<QnnExecuTorchBackendType, std::weak_ptr<QnnBackendBundle>>
113+
std::map<
114+
std::pair<QnnExecuTorchBackendType, QcomChipset>,
115+
std::weak_ptr<QnnBackendBundle>>
115116
qnn_backend_bundles_map_;
116117

117118
std::mutex mutex_; // Protects access to resources and ensures atomic

‎backends/qualcomm/runtime/backends/QnnFunctionInterface.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
*/
88
#pragma once
99

10+
#include <executorch/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h>
11+
1012
#include "QnnInterface.h"
1113
#include "Saver/QnnSaver.h"
1214

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Copyright (c) Qualcomm Innovation Center, Inc.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
#pragma once
9+
10+
#include "QnnCommon.h"
11+
12+
// QnnCommon.h exposes the QNN API ABI version, not the QAIRT SDK product
13+
// release. Use this helper for compile-time header/API availability checks.
14+
#define QNN_EXECUTORCH_QNN_API_VERSION_AT_LEAST(major, minor) \
15+
((QNN_API_VERSION_MAJOR > (major)) || \
16+
(QNN_API_VERSION_MAJOR == (major) && QNN_API_VERSION_MINOR >= (minor)))

‎backends/qualcomm/runtime/backends/QnnSysFunctionInterface.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
*/
88
#pragma once
99

10+
#include <executorch/backends/qualcomm/runtime/backends/QnnSdkCompatibility.h>
11+
1012
#include "System/QnnSystemInterface.h"
1113

1214
#include <utility>

0 commit comments

Comments
 (0)