Skip to content

Commit c55c31b

Browse files
Qualcomm AI Engine Direct - Add HTP context graph splitting option (pytorch#22595)
## Summary - Exposes QNN's HTP context graph-splitting config (`QNN_HTP_CONTEXT_CONFIG_OPTION_GRAPH_SPLITTING_CONFIGS` / `QnnHtpContext_GraphSplit_t`) via a new `use_graph_splitting` option on `generate_htp_compiler_spec()`, see https://docs.qualcomm.com/doc/80-63442-10/topic/htp_backend.html. This option significantly reduces finalization time at the cost of a slight increase in execution time. ### Misc - Wired up for the offline-prepare (host/x86 export) path only, since this feature is used in finalize. - Gated behind `QNN_HTP_API_VERSION_MAJOR >= 5 && QNN_HTP_API_VERSION_MINOR >= 49` since this option doesn't exist in older QNN_HTP_API_VERSION ## Test plan ``` python backends/qualcomm/tests/test_qnn_delegate.py -k "test_qnn_backend_graph_splitting" --device ef5e4029 --host localhost --soc_model SM8850 --build_folder build-android --executorch_root . ``` LLAMA3.2 3B performance: | Graph | Graph Preparation Initializing (us) | Graph Optimizations (us) | Post Graph Optimization (us) | Graph Sequencing for Target (us) | VTCM Allocation (us) | Parallelization Optimization (us) | Finalizing Graph Sequence (us) | Completion (us) | QNN (execute) time (us) | |---|---|---|---|---|---|---|---|---|---| | kv_forward1_without_gpe | 1146 | 26843138 | 433549 | 6828982 | 449030 | 5929118 | 266695 | 15171 | 9552 | | kv_forward1_with_gpe | 496 | 2014382 | 26630 | 427743 | 27504 | 246911 | 22066 | 1268 | 9880 | | kv_forward2_without_gpe | 373 | 27618966 | 1049136 | 10700233 | 313808 | 7947260 | 291464 | 14841 | 9452 | | kv_forward2_with_gpe | 599 | 2511473 | 34099 | 577162 | 36742 | 353654 | 31248 | 1754 | 9787 | | kv_forward3_without_gpe | 405 | 30262309 | 541066 | 8523872 | 407082 | 8058194 | 314898 | 16692 | 11136 | | kv_forward3_with_gpe | 512 | 5575498 | 75421 | 1322133 | 111640 | 1009982 | 51157 | 3371 | 11437 | | kv_forward4_without_gpe | 1966 | 10304161 | 36241 | 1151113 | 87287 | 749097 | 34706 | 1944 | 6703 | | kv_forward4_with_gpe | 342 | 8387236 | 41997 | 1104533 | 96270 | 870457 | 31896 | 2245 | 6686 | | prefill_forward1_without_gpe | 403 | 23425582 | 521963 | 20288202 | 28141820 | 13415019 | 534725 | 19491 | 23283 | | prefill_forward1_with_gpe | 635 | 2404414 | 42852 | 928676 | 607167 | 368336 | 25194 | 1790 | 20824 | | prefill_forward2_without_gpe | 354 | 22249865 | 496426 | 16837420 | 24738364 | 9634081 | 464298 | 16200 | 20473 | | prefill_forward2_with_gpe | 557 | 2521494 | 41107 | 929431 | 765368 | 426039 | 28336 | 1955 | 20764 | | prefill_forward3_without_gpe | 744 | 31356868 | 553758 | 17889195 | 17232990 | 9485483 | 500217 | 15025 | 21125 | | prefill_forward3_with_gpe | 1933 | 5689220 | 99188 | 2404130 | 2256161 | 1236475 | 79918 | 6054 | 21835 | | prefill_forward4_without_gpe | 889 | 10117697 | 43571 | 1450046 | 1323893 | 1062400 | 35717 | 2122 | 9431 | | prefill_forward4_with_gpe | 331 | 8484639 | 50456 | 1647489 | 1798310 | 1306534 | 45152 | 2760 | 9446 | This table is created by extracting data from finalization log: <img width="392" height="351" alt="image" src="https://github.com/user-attachments/assets/76f876f3-5084-4ac3-8be6-54b7971402e8" /> This PR was authored with assistance from Claude Code (Anthropic). cc @cbilgin @psiddh
1 parent c9595a6 commit c55c31b

8 files changed

Lines changed: 179 additions & 0 deletions

File tree

‎backends/qualcomm/runtime/backends/QnnBackendFactory.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ std::unique_ptr<BackendConfigParameters> QnnBackendFactory::Create(
6060
QNN_EXECUTORCH_LOG_INFO(
6161
"use_weight_sharing in htp_options: %d",
6262
htp_options->use_weight_sharing());
63+
QNN_EXECUTORCH_LOG_INFO(
64+
"use_graph_splitting in htp_options: %d",
65+
htp_options->use_graph_splitting());
6366
}
6467
backend_params->qnn_backend_cache_ptr_ =
6568
std::make_unique<HtpBackendCache>(

‎backends/qualcomm/runtime/backends/htp/host/HtpContextCustomConfig.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,29 @@ HtpContextCustomConfig::CreateContextCustomConfig() {
2626
ret.push_back(static_cast<QnnContext_CustomConfig_t>(p_custom_config));
2727
}
2828

29+
#if ( \
30+
QNN_HTP_API_VERSION_MAJOR > 5 || \
31+
(QNN_HTP_API_VERSION_MAJOR == 5 && QNN_HTP_API_VERSION_MINOR >= 49))
32+
if (htp_options_->use_graph_splitting()) {
33+
p_custom_config = AllocContextCustomConfig();
34+
p_custom_config->option =
35+
QNN_HTP_CONTEXT_CONFIG_OPTION_GRAPH_SPLITTING_CONFIGS;
36+
QnnHtpContext_GraphSplit_t graph_split_info{};
37+
graph_split_info.graphSplittingEnabled = true;
38+
p_custom_config->graphSplittingConfigs = graph_split_info;
39+
ret.push_back(static_cast<QnnContext_CustomConfig_t>(p_custom_config));
40+
}
41+
#else
42+
if (htp_options_->use_graph_splitting()) {
43+
QNN_EXECUTORCH_LOG_WARN(
44+
"use_graph_splitting is enabled but the QNN SDK used for this build "
45+
"(API %d.%d) does not support graph splitting, which requires HTP API "
46+
"5.49 or newer. The option will be ignored.",
47+
QNN_HTP_API_VERSION_MAJOR,
48+
QNN_HTP_API_VERSION_MINOR);
49+
}
50+
#endif
51+
2952
return ret;
3053
}
3154

‎backends/qualcomm/serialization/qc_compiler_spec.fbs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,10 @@ table QnnExecuTorchHtpBackendOptions {
206206
/// It will help the by reducing overall bandwith on the use case.
207207
/// The feature is only supported by specific SOCs.
208208
use_slc_allocator:bool;
209+
210+
/// When enabled, the compiled graph is split based on its structure and
211+
/// each part is compiled as an independent subgraph.
212+
use_graph_splitting:bool;
209213
}
210214

211215
/// Real-time: Indicates that the model is intended for real-time use cases, where a specific performance threshold must be met.

‎backends/qualcomm/serialization/qc_schema.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ class QnnExecuTorchHtpBackendOptions:
203203
use_multi_contexts: bool = False
204204
use_weight_sharing: bool = False
205205
use_slc_allocator: bool = False
206+
use_graph_splitting: bool = False
206207

207208

208209
@unique

‎backends/qualcomm/tests/rework/htp/feature/v68/test.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,16 @@
1212
from executorch.backends.qualcomm.tests.rework.src.feature import * # noqa: F403
1313

1414

15+
@pytest.mark.parametrize(
16+
"kwargs",
17+
[
18+
pytest.param({"expected": Tolerance()}, id="e2e"),
19+
],
20+
)
21+
def test_graph_splitting(request, kwargs):
22+
GraphSplit.test_graph_splitting(request, kwargs) # noqa: F405
23+
24+
1525
@pytest.mark.parametrize(
1626
"kwargs",
1727
[

‎backends/qualcomm/tests/rework/src/feature.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,81 @@ def get_quantizer(qnn_config: QnnConfig):
7171
)
7272

7373

74+
class GraphSplit:
75+
class Model(torch.nn.Module):
76+
def __init__(self):
77+
super().__init__()
78+
79+
def example_inputs(self):
80+
return (torch.randn(1, 2, 3, 4),)
81+
82+
def forward(self, x):
83+
return torch.nn.ReLU()(x)
84+
85+
@staticmethod
86+
def _test(qnn_config, compile_specs, expected):
87+
def callback(adb: SimpleADB, pattern):
88+
def verify(log):
89+
msg = log.stdout
90+
assert pattern in msg, f"{pattern} in log"
91+
92+
# QnnExecuTorchLogLevel.kLogLevelVerbose
93+
adb.extra_cmds += " --log_level 4"
94+
adb.execute(output_callback=verify)
95+
96+
with expected:
97+
# model declaration
98+
model = __class__.Model()
99+
inputs = model.example_inputs()
100+
# perform ptq
101+
with calibrate(
102+
model, [inputs], make_quantizer(soc_model=qnn_config.soc_model)
103+
) as model:
104+
executorch_prog_mgr = to_edge_transform_and_lower_to_qnn(
105+
module=model,
106+
inputs=inputs,
107+
compiler_specs=compile_specs,
108+
).to_executorch()
109+
# file for subgraph 0
110+
# delete artifact before assertion to avoid leak
111+
file_name = "forward_schematic.bin_sg_0.py"
112+
file_exist = os.path.isfile(file_name)
113+
if file_exist:
114+
os.remove(file_name)
115+
assert (
116+
file_exist
117+
), f"{file_name} not produced; graph splitting did not run"
118+
# remote testing
119+
invoke_remote(
120+
qnn_config=qnn_config,
121+
executorch_prog=executorch_prog_mgr,
122+
callback=partial(callback, pattern="Found blob with gpe enabled"),
123+
)
124+
125+
@staticmethod
126+
@unpack_fixtures
127+
def test_graph_splitting(qnn_config, compile_specs, expected):
128+
# extend this for other backends
129+
backend_compile_specs = {
130+
QnnExecuTorchBackendType.kHtpBackend: compile_specs(
131+
tuple(
132+
{
133+
"soc_model": getattr(QcomChipset, qnn_config.soc_model),
134+
"use_graph_splitting": True,
135+
"use_fp16": False,
136+
"profile_level": 3,
137+
}.items()
138+
)
139+
),
140+
}
141+
142+
__class__._test(
143+
qnn_config=qnn_config,
144+
compile_specs=backend_compile_specs[qnn_config.backend],
145+
expected=expected,
146+
)
147+
148+
74149
class Logging:
75150
class Model(torch.nn.Module):
76151
def __init__(self):

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7248,6 +7248,36 @@ def test_qnn_backend_multi_contexts_composite(self):
72487248
exec_prog = edge_prog.to_executorch()
72497249
self.verify_output(module.get_reference_module(), sample_input, exec_prog)
72507250

7251+
@unittest.skipIf(
7252+
is_qnn_sdk_version_less_than("2.49"),
7253+
"feature is enable after 2.49.",
7254+
)
7255+
def test_qnn_backend_graph_splitting(self):
7256+
backend_options = generate_htp_compiler_spec(
7257+
use_fp16=True,
7258+
use_graph_splitting=True,
7259+
)
7260+
compiler_spec = generate_qnn_executorch_compiler_spec(
7261+
soc_model=self.chipset_table[TestQNN.soc_model],
7262+
backend_options=backend_options,
7263+
profile_level=3,
7264+
)
7265+
sample_input = (torch.randn([2, 5, 1, 3]),)
7266+
module = Relu() # noqa: F405
7267+
edge_prog_mgr = to_edge_transform_and_lower_to_qnn(
7268+
module, sample_input, compiler_spec
7269+
).to_executorch()
7270+
# file for subgraph 0
7271+
# delete artifact before assertion to avoid leak
7272+
file_name = "forward_schematic.bin_sg_0.py"
7273+
file_exist = os.path.isfile(file_name)
7274+
if file_exist:
7275+
os.remove(file_name)
7276+
self.assertTrue(file_exist)
7277+
self.verify_output(
7278+
module=module, sample_inputs=sample_input, executorch_prog=edge_prog_mgr
7279+
)
7280+
72517281
def test_qnn_backend_multi_graphs(self):
72527282
if self.enable_x86_64:
72537283
self.skipTest("weight sharing is not supported on host machine")
@@ -8326,6 +8356,35 @@ def test_qnn_backend_multi_contexts_composite(self):
83268356
exec_prog = edge_prog.to_executorch()
83278357
self.verify_output(module.get_reference_module(), sample_input, exec_prog)
83288358

8359+
@unittest.skipIf(
8360+
is_qnn_sdk_version_less_than("2.49"),
8361+
"feature is enable after 2.49.",
8362+
)
8363+
def test_qnn_backend_graph_splitting(self):
8364+
backend_options = generate_htp_compiler_spec(
8365+
use_fp16=False,
8366+
use_graph_splitting=True,
8367+
)
8368+
compiler_spec = generate_qnn_executorch_compiler_spec(
8369+
soc_model=self.chipset_table[TestQNN.soc_model],
8370+
backend_options=backend_options,
8371+
profile_level=3,
8372+
)
8373+
sample_input = (torch.randn([2, 5, 1, 3]),)
8374+
module = Relu() # noqa: F405
8375+
module = self.get_qdq_module(
8376+
module, sample_input, quant_dtype=QuantDtype.use_8a8w
8377+
)
8378+
edge_prog_mgr = to_edge_transform_and_lower_to_qnn(
8379+
module, sample_input, compiler_spec
8380+
).to_executorch()
8381+
# file for subgraph 0
8382+
self.assertTrue(os.path.isfile("forward_schematic.bin_sg_0.py"))
8383+
os.remove("forward_schematic.bin_sg_0.py")
8384+
self.verify_output(
8385+
module=module, sample_inputs=sample_input, executorch_prog=edge_prog_mgr
8386+
)
8387+
83298388
def test_qnn_backend_multi_graphs(self):
83308389
if self.enable_x86_64:
83318390
self.skipTest("weight sharing is not supported on host machine")

‎backends/qualcomm/utils/utils.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1094,6 +1094,7 @@ def generate_htp_compiler_spec(
10941094
use_multi_contexts: bool = False,
10951095
use_weight_sharing: bool = False,
10961096
use_slc_allocator: bool = False,
1097+
use_graph_splitting: bool = False,
10971098
htp_performance_mode: QnnExecuTorchHtpPerformanceMode = QnnExecuTorchHtpPerformanceMode.kHtpBurst,
10981099
) -> QnnExecuTorchBackendOptions:
10991100
"""
@@ -1113,6 +1114,8 @@ def generate_htp_compiler_spec(
11131114
use_slc_allocator: Allows user to enable the usage of the System Level Cache Allocator for a given graph.
11141115
It will help the by reducing overall bandwith on the use case.
11151116
The feature is only supported by specific SOCs.
1117+
use_graph_splitting: When enabled, the compiled graph is split based on
1118+
its structure and each part is compiled as an independent subgraph.
11161119
11171120
Returns:
11181121
QnnExecuTorchHtpBackendOptions: backend options for QNN HTP.
@@ -1131,6 +1134,7 @@ def generate_htp_compiler_spec(
11311134
htp_options.use_weight_sharing = use_weight_sharing
11321135
htp_options.use_dlbc = use_dlbc
11331136
htp_options.use_slc_allocator = use_slc_allocator
1137+
htp_options.use_graph_splitting = use_graph_splitting
11341138
return QnnExecuTorchBackendOptions(
11351139
backend_type=QnnExecuTorchBackendType.kHtpBackend,
11361140
htp_options=htp_options,

0 commit comments

Comments
 (0)