Skip to content

Commit ef01117

Browse files
authored
xnnpack backend support in devtool example runner (pytorch#16812)
cmake and build script update to support xnnpack backend in devtool runner. Also update numeric discrepancy tutorial to use that.
1 parent b89dc1a commit ef01117

3 files changed

Lines changed: 132 additions & 96 deletions

File tree

‎docs/source/tutorials_source/devtools-debugging-tutorial.py‎

Lines changed: 112 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -65,51 +65,47 @@
6565
# model graphs and metadata for linking runtime results to the eager model.
6666
# We use ``to_edge_transform_and_lower`` with ``generate_etrecord=True`` to
6767
# automatically capture the ETRecord during the lowering process.
68-
69-
import os
70-
import tempfile
71-
72-
import torch
73-
74-
from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
75-
from executorch.backends.xnnpack.utils.configs import get_xnnpack_edge_compile_config
76-
77-
from executorch.exir import ExecutorchProgramManager, to_edge_transform_and_lower
78-
from torch.export import export, ExportedProgram
79-
from torchvision import models # type: ignore[import-untyped]
80-
81-
# Create Vision Transformer model
82-
vit = models.vision_transformer.vit_b_16(weights="IMAGENET1K_V1")
83-
model = vit.eval()
84-
model_inputs = (torch.randn(1, 3, 224, 224),)
85-
86-
temp_dir = tempfile.mkdtemp()
87-
88-
# Export and lower model to XNNPACK delegate
89-
aten_model: ExportedProgram = export(model, model_inputs, strict=True)
90-
edge_program_manager = to_edge_transform_and_lower(
91-
aten_model,
92-
partitioner=[XnnpackPartitioner()],
93-
compile_config=get_xnnpack_edge_compile_config(),
94-
generate_etrecord=True,
95-
)
96-
97-
et_program_manager: ExecutorchProgramManager = edge_program_manager.to_executorch()
98-
99-
# Save the .pte file
100-
pte_path = os.path.join(temp_dir, "model.pte")
101-
et_program_manager.save(pte_path)
102-
103-
# Get and save ETRecord with representative inputs
104-
etrecord = et_program_manager.get_etrecord()
105-
etrecord.update_representative_inputs(model_inputs)
106-
etrecord_path = os.path.join(temp_dir, "etrecord.bin")
107-
etrecord.save(etrecord_path)
108-
109-
# sphinx_gallery_start_ignore
110-
from unittest.mock import patch
111-
112-
# sphinx_gallery_end_ignore
68+
#
69+
# .. code-block:: python
70+
#
71+
# import os
72+
# import tempfile
73+
#
74+
# import torch
75+
# from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
76+
# from executorch.backends.xnnpack.utils.configs import get_xnnpack_edge_compile_config
77+
# from executorch.exir import ExecutorchProgramManager, to_edge_transform_and_lower
78+
# from torch.export import export, ExportedProgram
79+
# from torchvision import models # type: ignore[import-untyped]
80+
#
81+
# # Create Vision Transformer model
82+
# vit = models.vision_transformer.vit_b_16(weights="IMAGENET1K_V1")
83+
# model = vit.eval()
84+
# model_inputs = (torch.randn(1, 3, 224, 224),)
85+
#
86+
# temp_dir = tempfile.mkdtemp()
87+
#
88+
# # Export and lower model to XNNPACK delegate
89+
# aten_model: ExportedProgram = export(model, model_inputs, strict=True)
90+
# edge_program_manager = to_edge_transform_and_lower(
91+
# aten_model,
92+
# partitioner=[XnnpackPartitioner()],
93+
# compile_config=get_xnnpack_edge_compile_config(),
94+
# generate_etrecord=True,
95+
# )
96+
#
97+
# et_program_manager: ExecutorchProgramManager = edge_program_manager.to_executorch()
98+
#
99+
# # Save the .pte file
100+
# pte_path = os.path.join(temp_dir, "model.pte")
101+
# et_program_manager.save(pte_path)
102+
#
103+
# # Get and save ETRecord with representative inputs
104+
# etrecord = et_program_manager.get_etrecord()
105+
# etrecord.update_representative_inputs(model_inputs)
106+
# etrecord_path = os.path.join(temp_dir, "etrecord.bin")
107+
# etrecord.save(etrecord_path)
108+
#
113109

114110
######################################################################
115111
#
@@ -226,7 +222,9 @@
226222
# |----|----------------------------------------------------------------|----------------------------------------------------|----------------------------------------------------|----------------------------------------------------| ---------------------------|
227223
# | 0 | [conv2d] | [[[tensor([-0.0130, 0.0075, -0.0334, -0.0122,... | [DELEGATE_CALL] | [[[tensor([-0.0130, 0.0075, -0.0334, -0.0122,... | [3.2530690555343034e-15] |
228224
# | 1 | [permute, cat, add, dropout] | [[[tensor(-0.0024), tensor(0.0054), tensor(0.0... | [DELEGATE_CALL] | [[[tensor(-0.0024), tensor(0.0054), tensor(0.0... | [3.2488685838924244e-15] |
225+
# ...
229226
# | 4 | [transpose, linear, unflatten, unsqueeze, tran...] | [[[tensor(0.0045), tensor(-0.0084), tensor(0.0... | [DELEGATE_CALL, DELEGATE_CALL, DELEGATE_CALL, ...] | [[tensor(0.0045), tensor(-0.0084), tensor(0.00... | [0.00010033142876115867] |
227+
# ...
230228
# | 59 | [transpose_66, linear_44, unflatten_11, unsque...] | [[[tensor(-0.3346), tensor(0.1540), tensor(-0.... | [DELEGATE_CALL, DELEGATE_CALL, DELEGATE_CALL, ...] | [[tensor(-0.3346), tensor(0.1540), tensor(-0.0... | [0.02629170972698486] |
231229
#
232230

@@ -285,31 +283,46 @@
285283
# Step 1: Export Model and Generate ETRecord
286284
# ------------------------------------------
287285
#
288-
# Same as Pipeline 1 - we reuse the model and export artifacts we already created.
289-
# The key artifact needed for the CMake pipeline is:
286+
# First, we export the model and generate an ``ETRecord``, same as step 1 of pipeline 1:
287+
#
288+
# .. code-block:: python
290289
#
291-
# - ``bundled_program.bpte``: The BundledProgram file contains the model and
292-
# sample inputs/outputs for testing.
290+
# import os
291+
# import tempfile
293292
#
294-
# Most of the pipeline were the same as Pipeline 1's Step 1. If you're only using
295-
# the CMake pipeline, use the same export code:
293+
# import torch
294+
# from executorch.backends.xnnpack.partition.xnnpack_partitioner import XnnpackPartitioner
295+
# from executorch.backends.xnnpack.utils.configs import get_xnnpack_edge_compile_config
296+
# from executorch.exir import ExecutorchProgramManager, to_edge_transform_and_lower
297+
# from torch.export import export, ExportedProgram
298+
# from torchvision import models # type: ignore[import-untyped]
296299
#
297-
# .. code-block:: python
300+
# # Create Vision Transformer model
301+
# vit = models.vision_transformer.vit_b_16(weights="IMAGENET1K_V1")
302+
# model = vit.eval()
303+
# model_inputs = (torch.randn(1, 3, 224, 224),)
298304
#
299-
# # Export and lower model (same as Pipeline 1)
300-
# aten_model = export(model, model_inputs, strict=True)
305+
# temp_dir = tempfile.mkdtemp()
306+
#
307+
# # Export and lower model to XNNPACK delegate
308+
# aten_model: ExportedProgram = export(model, model_inputs, strict=True)
301309
# edge_program_manager = to_edge_transform_and_lower(
302310
# aten_model,
303311
# partitioner=[XnnpackPartitioner()],
304312
# compile_config=get_xnnpack_edge_compile_config(),
305313
# generate_etrecord=True,
306314
# )
307-
# et_program_manager = edge_program_manager.to_executorch()
308315
#
309-
# # Save artifacts
316+
# et_program_manager: ExecutorchProgramManager = edge_program_manager.to_executorch()
317+
#
318+
# # Save the .pte file
319+
# pte_path = os.path.join(temp_dir, "model.pte")
310320
# et_program_manager.save(pte_path)
321+
#
322+
# # Get and save ETRecord with representative inputs
311323
# etrecord = et_program_manager.get_etrecord()
312324
# etrecord.update_representative_inputs(model_inputs)
325+
# etrecord_path = os.path.join(temp_dir, "etrecord.bin")
313326
# etrecord.save(etrecord_path)
314327
#
315328

@@ -329,15 +342,17 @@
329342
# serialize_from_bundled_program_to_flatbuffer,
330343
# )
331344
#
332-
# # Construct Method Test Suites using the same model and inputs from Pipeline 1
345+
# # Define the method name and test inputs
346+
# # IMPORTANT: Use the same inputs as etrecord.update_representative_inputs()
333347
# m_name = "forward"
334-
# inputs = [model_inputs for _ in range(2)]
348+
# test_inputs = [model_inputs]
335349
#
350+
# # Create test cases by running the eager model to get expected outputs
336351
# method_test_suites = [
337352
# MethodTestSuite(
338353
# method_name=m_name,
339354
# test_cases=[
340-
# MethodTestCase(inputs=inp, expected_outputs=model(*inp)) for inp in inputs
355+
# MethodTestCase(inputs=inp, expected_outputs=model(*inp)) for inp in test_inputs
341356
# ],
342357
# )
343358
# ]
@@ -355,26 +370,37 @@
355370
#
356371

357372
######################################################################
358-
# Step 3: Run with CMake Example Runner
373+
# Step 3: Run with Devtool Example Runner
359374
# -------------------------------------
360375
#
361-
# Build and run the example runner with output verification and debug output enabled::
376+
# This step we will verify the final result and generate etdump for next
377+
# step usage by using devtool example runner.
378+
#
379+
# First, build the example runner with XNNPACK backend support:
380+
#
381+
# .. code-block:: bash
382+
#
383+
# cd /path/to/executorch
384+
# ./examples/devtools/build_example_runner.sh --xnnpack
362385
#
363-
# cd executorch
364-
# ./examples/devtools/build_example_runner.sh
365-
# cmake-out/examples/devtools/example_runner \
366-
# --bundled_program_path="bundled_program.bpte" \
367-
# --output_verification=true \
368-
# --dump_intermediate_outputs=true
386+
# where ``--xnnpack`` is a build flag that enables XNNPACK backend support.
387+
# Then run the example runner with output verification and debug output enabled:
388+
#
389+
# .. code-block:: bash
390+
#
391+
# cmake-out/examples/devtools/example_runner \
392+
# --bundled_program_path=/path/to/bundled_program.bpte \
393+
# --output_verification \
394+
# --dump_intermediate_outputs \
395+
# --debug_buffer_size=1073741824
369396
#
370397
# The key flags are:
371398
#
372-
# - ``--output_verification=true``: Compare runtime outputs against the expected
399+
# - ``--output_verification``: Compare runtime outputs against the expected
373400
# outputs stored in the BundledProgram (uses rtol=1e-3, atol=1e-5)
374-
# - ``--dump_intermediate_outputs=true``: Capture intermediate outputs for
401+
# - ``--dump_intermediate_outputs``: Capture intermediate outputs for
375402
# operator-level debugging
376-
# - ``--debug_buffer_size=<bytes>``: Size of debug buffer (default: 256KB, increase
377-
# for larger models)
403+
# - ``--debug_buffer_size=<bytes>``: Size of debug buffer (1GB in this example)
378404
#
379405
# Example output on success:
380406
#
@@ -402,28 +428,21 @@
402428
#
403429
# After running the model with the CMake runner, load the generated artifacts
404430
# back into Python for analysis using the Inspector.
405-
406-
from executorch.devtools import Inspector
407-
408-
# sphinx_gallery_start_ignore
409-
inspector_patch = patch.object(Inspector, "__init__", return_value=None)
410-
inspector_patch.start()
411-
# sphinx_gallery_end_ignore
412-
etrecord_path = "etrecord.bin"
413-
etdump_path = "etdump.etdp"
414-
debug_buffer_path = "debug_output.bin"
415-
416-
inspector = Inspector(
417-
etdump_path=etdump_path,
418-
etrecord=etrecord_path,
419-
debug_buffer_path=debug_buffer_path,
420-
)
421-
422-
# sphinx_gallery_start_ignore
423-
inspector_patch.stop()
424-
# sphinx_gallery_end_ignore
425-
426-
######################################################################
431+
#
432+
# .. code-block:: python
433+
#
434+
# from executorch.devtools import Inspector
435+
#
436+
# etrecord_path = "/path/to/etrecord.bin"
437+
# etdump_path = "/path/to/etdump.etdp"
438+
# debug_buffer_path = "/path/to/debug_output.bin"
439+
#
440+
# inspector = Inspector(
441+
# etdump_path=etdump_path,
442+
# etrecord=etrecord_path,
443+
# debug_buffer_path=debug_buffer_path,
444+
# )
445+
#
427446
# Then use the same analysis techniques as in Pipeline 1:
428447
#
429448
# .. code-block:: python

‎examples/devtools/CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ cmake_minimum_required(VERSION 3.19)
1212
project(devtools_example)
1313

1414
option(EXECUTORCH_BUILD_COREML "Build the Core ML backend" OFF)
15+
option(EXECUTORCH_BUILD_XNNPACK "Build the XNNPACK backend" OFF)
1516

1617
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
1718
if(NOT CMAKE_CXX_STANDARD)
@@ -95,3 +96,14 @@ if(EXECUTORCH_BUILD_COREML)
9596
${FOUNDATION_FRAMEWORK} ${SQLITE_LIBRARY}
9697
)
9798
endif()
99+
100+
if(EXECUTORCH_BUILD_XNNPACK)
101+
if(TARGET xnnpack_backend)
102+
set(xnnpack_backend_libs xnnpack_backend XNNPACK xnnpack-microkernels-prod)
103+
if(TARGET kleidiai)
104+
list(APPEND xnnpack_backend_libs kleidiai)
105+
endif()
106+
target_link_libraries(example_runner ${xnnpack_backend_libs})
107+
executorch_target_link_options_shared_lib(xnnpack_backend)
108+
endif()
109+
endif()

‎examples/devtools/build_example_runner.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,21 @@ readonly EXECUTORCH_ROOT="${SCRIPT_DIR}/../.."
1818
export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-9}"
1919

2020
BUILD_COREML=OFF
21+
BUILD_XNNPACK=OFF
2122

2223
usage() {
2324
echo "Builds example runner."
2425
echo "Options:"
2526
echo " --coreml Include this flag to enable Core ML backend when building the Developer Tools."
27+
echo " --xnnpack Include this flag to enable XNNPACK backend when building the Developer Tools."
2628
exit 0
2729
}
2830

2931
for arg in "$@"; do
3032
case $arg in
3133
-h|--help) usage ;;
3234
--coreml) BUILD_COREML=ON ;;
35+
--xnnpack) BUILD_XNNPACK=ON ;;
3336
*)
3437
esac
3538
done
@@ -39,19 +42,20 @@ main() {
3942

4043
./install_executorch.sh --clean
4144

42-
if [[ "${BUILD_COREML}" == "ON" ]]; then
45+
if [[ "${BUILD_COREML}" == "ON" ]] || [[ "${BUILD_XNNPACK}" == "ON" ]]; then
4346
cmake -DCMAKE_INSTALL_PREFIX=cmake-out \
4447
-DCMAKE_BUILD_TYPE=Release \
4548
-DEXECUTORCH_BUILD_DEVTOOLS=ON \
4649
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
47-
-DEXECUTORCH_BUILD_COREML=ON \
50+
-DEXECUTORCH_BUILD_COREML=$BUILD_COREML \
51+
-DEXECUTORCH_BUILD_XNNPACK=$BUILD_XNNPACK \
4852
-Dprotobuf_BUILD_TESTS=OFF \
4953
-Dprotobuf_BUILD_EXAMPLES=OFF \
5054
-DEXECUTORCH_ENABLE_LOGGING=ON \
5155
-Bcmake-out .
5256
else
5357
cmake -DCMAKE_INSTALL_PREFIX=cmake-out \
54-
-DCMAKE_BUILD_TYPE=Release \
58+
-DCMAKE_BUILD_TYPE=Debug \
5559
-DEXECUTORCH_BUILD_DEVTOOLS=ON \
5660
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
5761
-Bcmake-out .
@@ -74,6 +78,7 @@ main() {
7478
cmake -DCMAKE_PREFIX_PATH="${cmake_prefix_path}" \
7579
-DCMAKE_BUILD_TYPE=Release \
7680
-DEXECUTORCH_BUILD_COREML=$BUILD_COREML \
81+
-DEXECUTORCH_BUILD_XNNPACK=$BUILD_XNNPACK \
7782
-B"${build_dir}" \
7883
"${example_dir}"
7984
cmake --build "${build_dir}" --config Release

0 commit comments

Comments
 (0)