From 8322397065f081d96897693b24217f07f00fad35 Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Sun, 18 Oct 2026 20:22:15 +0200 Subject: [PATCH 01/10] Add ESP-NN backend for Espressif ESP32 / ESP32-S3 EspNNBackend lowers PT2E int8 subgraphs to ESP-NN kernels; unsupported ops fall back to ExecuTorch portable kernels. Adds the ESP-IDF runtime component, an on-device test runner, a camera demo and ESP32-S3 / ESP32 benchmarks, alongside the existing examples/espressif executor runner. --- backends/espressif/README.md | 350 +++ backends/espressif/__init__.py | 16 + backends/espressif/_passes/__init__.py | 12 + backends/espressif/_passes/fold_batch_norm.py | 79 + backends/espressif/_passes/remove_dropout.py | 45 + backends/espressif/espnn_preprocess.py | 58 + .../espressif/espnn_runtime/CMakeLists.txt | 96 + backends/espressif/espnn_runtime/Kconfig | 39 + .../espressif/espnn_runtime/idf_component.yml | 13 + .../include/espnn/espnn_backend.h | 57 + .../include/espnn/int8_interpreter.h | 235 ++ .../espnn_runtime/include/espnn/xts3_graph.h | 240 ++ .../espnn_runtime/src/espnn_backend.cpp | 281 +++ .../espressif/espnn_runtime/src/espnn_pal.cpp | 59 + .../espnn_runtime/src/int8_interpreter.cpp | 1767 ++++++++++++++ .../espnn_runtime/src/xts3_graph.cpp | 1050 +++++++++ backends/espressif/lowering.py | 36 + backends/espressif/partition/__init__.py | 11 + .../espressif/partition/espnn_partitioner.py | 423 ++++ backends/espressif/quantizer/__init__.py | 11 + .../espressif/quantizer/espnn_quantizer.py | 372 +++ backends/espressif/reference/__init__.py | 12 + .../espressif/reference/esp_nn_reference.py | 643 ++++++ .../espressif/reference/program_emulator.py | 98 + backends/espressif/serialization/__init__.py | 11 + backends/espressif/serialization/int8_ops.py | 689 ++++++ backends/espressif/serialization/xts3.py | 2048 +++++++++++++++++ backends/espressif/test/__init__.py | 7 + backends/espressif/test/test_cpu_fallback.py | 169 ++ .../espressif/test/test_fold_batch_norm.py | 85 + backends/espressif/test/test_int8_lowering.py | 336 +++ backends/espressif/test/test_int8_ops.py | 299 +++ examples/espressif/.gitignore | 10 + examples/espressif/aot_espnn_compiler.py | 518 +++++ examples/espressif/benchmarks/esp32.json | 186 ++ examples/espressif/benchmarks/esp32.md | 17 + examples/espressif/benchmarks/esp32s3.json | 370 +++ examples/espressif/benchmarks/esp32s3.md | 20 + .../espressif/camera_stream/CMakeLists.txt | 17 + examples/espressif/camera_stream/README.md | 30 + examples/espressif/camera_stream/flash.sh | 20 + .../camera_stream/imagenet_classes.txt | 1000 ++++++++ .../camera_stream/main/CMakeLists.txt | 42 + .../camera_stream/main/Kconfig.projbuild | 77 + .../camera_stream/main/camera_stream.cpp | 994 ++++++++ .../camera_stream/main/camera_stream.h | 26 + .../camera_stream/main/idf_component.yml | 12 + .../espressif/camera_stream/main/main.cpp | 28 + .../espressif/camera_stream/main/model_data.S | 20 + .../espressif/camera_stream/partitions.csv | 3 + .../camera_stream/sdkconfig.defaults | 18 + examples/espressif/camera_stream/viewer.py | 789 +++++++ examples/espressif/espnn_bundle.py | 153 ++ .../espressif/espnn_runner/CMakeLists.txt | 17 + examples/espressif/espnn_runner/README.md | 49 + .../espnn_runner/main/CMakeLists.txt | 45 + .../espnn_runner/main/Kconfig.projbuild | 51 + .../espressif/espnn_runner/main/bundle_data.S | 20 + .../espnn_runner/main/int8_suite.cpp | 324 +++ examples/espressif/espnn_runner/main/main.cpp | 89 + .../espressif/espnn_runner/main/pte_suite.cpp | 494 ++++ examples/espressif/espnn_runner/main/runner.h | 48 + .../espnn_runner/partitions_16mb.csv | 3 + .../espressif/espnn_runner/partitions_4mb.csv | 3 + .../espressif/espnn_runner/sdkconfig.defaults | 11 + .../espnn_runner/sdkconfig.defaults.esp32 | 5 + .../espnn_runner/sdkconfig.defaults.esp32s3 | 9 + .../espnn_runner/sdkconfig.executorch | 1 + examples/espressif/quantize_and_lower.py | 54 + examples/espressif/scripts/bench_models.py | 195 ++ examples/espressif/scripts/benchmark.py | 520 +++++ examples/espressif/scripts/board_dashboard.py | 308 +++ .../espressif/scripts/build_executorch_esp.sh | 80 + examples/espressif/scripts/espnn_board.py | 297 +++ examples/espressif/scripts/license_headers.py | 140 ++ .../espressif/scripts/make_test_bundles.py | 169 ++ examples/espressif/scripts/run_bundle.py | 79 + 77 files changed, 17008 insertions(+) create mode 100644 backends/espressif/README.md create mode 100644 backends/espressif/__init__.py create mode 100644 backends/espressif/_passes/__init__.py create mode 100644 backends/espressif/_passes/fold_batch_norm.py create mode 100644 backends/espressif/_passes/remove_dropout.py create mode 100644 backends/espressif/espnn_preprocess.py create mode 100644 backends/espressif/espnn_runtime/CMakeLists.txt create mode 100644 backends/espressif/espnn_runtime/Kconfig create mode 100644 backends/espressif/espnn_runtime/idf_component.yml create mode 100644 backends/espressif/espnn_runtime/include/espnn/espnn_backend.h create mode 100644 backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h create mode 100644 backends/espressif/espnn_runtime/include/espnn/xts3_graph.h create mode 100644 backends/espressif/espnn_runtime/src/espnn_backend.cpp create mode 100644 backends/espressif/espnn_runtime/src/espnn_pal.cpp create mode 100644 backends/espressif/espnn_runtime/src/int8_interpreter.cpp create mode 100644 backends/espressif/espnn_runtime/src/xts3_graph.cpp create mode 100644 backends/espressif/lowering.py create mode 100644 backends/espressif/partition/__init__.py create mode 100644 backends/espressif/partition/espnn_partitioner.py create mode 100644 backends/espressif/quantizer/__init__.py create mode 100644 backends/espressif/quantizer/espnn_quantizer.py create mode 100644 backends/espressif/reference/__init__.py create mode 100644 backends/espressif/reference/esp_nn_reference.py create mode 100644 backends/espressif/reference/program_emulator.py create mode 100644 backends/espressif/serialization/__init__.py create mode 100644 backends/espressif/serialization/int8_ops.py create mode 100644 backends/espressif/serialization/xts3.py create mode 100644 backends/espressif/test/__init__.py create mode 100644 backends/espressif/test/test_cpu_fallback.py create mode 100644 backends/espressif/test/test_fold_batch_norm.py create mode 100644 backends/espressif/test/test_int8_lowering.py create mode 100644 backends/espressif/test/test_int8_ops.py create mode 100644 examples/espressif/.gitignore create mode 100644 examples/espressif/aot_espnn_compiler.py create mode 100644 examples/espressif/benchmarks/esp32.json create mode 100644 examples/espressif/benchmarks/esp32.md create mode 100644 examples/espressif/benchmarks/esp32s3.json create mode 100644 examples/espressif/benchmarks/esp32s3.md create mode 100644 examples/espressif/camera_stream/CMakeLists.txt create mode 100644 examples/espressif/camera_stream/README.md create mode 100755 examples/espressif/camera_stream/flash.sh create mode 100644 examples/espressif/camera_stream/imagenet_classes.txt create mode 100644 examples/espressif/camera_stream/main/CMakeLists.txt create mode 100644 examples/espressif/camera_stream/main/Kconfig.projbuild create mode 100644 examples/espressif/camera_stream/main/camera_stream.cpp create mode 100644 examples/espressif/camera_stream/main/camera_stream.h create mode 100644 examples/espressif/camera_stream/main/idf_component.yml create mode 100644 examples/espressif/camera_stream/main/main.cpp create mode 100644 examples/espressif/camera_stream/main/model_data.S create mode 100644 examples/espressif/camera_stream/partitions.csv create mode 100644 examples/espressif/camera_stream/sdkconfig.defaults create mode 100644 examples/espressif/camera_stream/viewer.py create mode 100644 examples/espressif/espnn_bundle.py create mode 100644 examples/espressif/espnn_runner/CMakeLists.txt create mode 100644 examples/espressif/espnn_runner/README.md create mode 100644 examples/espressif/espnn_runner/main/CMakeLists.txt create mode 100644 examples/espressif/espnn_runner/main/Kconfig.projbuild create mode 100644 examples/espressif/espnn_runner/main/bundle_data.S create mode 100644 examples/espressif/espnn_runner/main/int8_suite.cpp create mode 100644 examples/espressif/espnn_runner/main/main.cpp create mode 100644 examples/espressif/espnn_runner/main/pte_suite.cpp create mode 100644 examples/espressif/espnn_runner/main/runner.h create mode 100644 examples/espressif/espnn_runner/partitions_16mb.csv create mode 100644 examples/espressif/espnn_runner/partitions_4mb.csv create mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults create mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults.esp32 create mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 create mode 100644 examples/espressif/espnn_runner/sdkconfig.executorch create mode 100644 examples/espressif/quantize_and_lower.py create mode 100644 examples/espressif/scripts/bench_models.py create mode 100644 examples/espressif/scripts/benchmark.py create mode 100644 examples/espressif/scripts/board_dashboard.py create mode 100755 examples/espressif/scripts/build_executorch_esp.sh create mode 100644 examples/espressif/scripts/espnn_board.py create mode 100644 examples/espressif/scripts/license_headers.py create mode 100644 examples/espressif/scripts/make_test_bundles.py create mode 100644 examples/espressif/scripts/run_bundle.py diff --git a/backends/espressif/README.md b/backends/espressif/README.md new file mode 100644 index 00000000000..5417bb5d169 --- /dev/null +++ b/backends/espressif/README.md @@ -0,0 +1,350 @@ +# ESP-NN backend for Espressif chips + +This backend runs int8-quantized PyTorch models on Espressif microcontrollers. +It is tuned for the **ESP32-S3** and also runs on the original ESP32. It +delegates quantized subgraphs to [ESP-NN](https://github.com/espressif/esp-nn), +Espressif's kernel library. ESP-NN has SIMD kernels for the ESP32-S3 and +portable C kernels for the other chips. + +It has two ways to run a model: + +| Path | What runs on the chip | When to use it | +|---|---|---| +| **ExecuTorch delegate** | A normal `.pte` program. `EspNNBackend` runs the int8 partitions with ESP-NN, and ExecuTorch's portable kernels run everything else (float ops, unsupported layers, pre- and post-processing). | Any model that `torch.export` can capture, including models that only partly map to ESP-NN. | +| **Standalone int8 graph** | The same delegate payload (an XTS3 graph), run directly by the small `espnn_runtime` interpreter with no ExecuTorch runtime linked. | Models that are fully int8 end to end, when firmware size matters (the runner is about 330 KB). | + +The two paths share one lowering flow: the PT2E quantizer, the partitioner and +the XTS3 serializer. The host reference interpreter reproduces the device +arithmetic, so every test and example checks the board's output bit-exactly. + +## Contents + +``` +backends/espressif/ + espnn_preprocess.py EspNNBackend: BackendDetails.preprocess -> XTS3 graph + partition/ EspNNPartitioner: int8 regions -> delegate, rest -> CPU + quantizer/ EspNNQuantizer: PT2E int8 annotations ESP-NN can execute + lowering.py to_espnn_edge(): export -> edge -> lowered program + report + serialization/ XTS3 format writer and parser, op lowering rules + reference/ bit-exact host model of the ESP-NN kernels and the CPU fallback + _passes/ batch-norm folding, dropout removal + espnn_runtime/ ESP-IDF component: XTS3 interpreter + ExecuTorch BackendInterface + test/ pytest suite (lowering, operators, CPU fallback) +examples/espressif/ + quantize_and_lower.py minimal quantize -> lower -> .pte example + aot_espnn_compiler.py full compiler CLI: calibration, QAT, accuracy, device test vectors + espnn_runner/ ESP-IDF app that runs and verifies bundles on a board + camera_stream/ live camera classification demo (ESP32-S3) + scripts/ cross-build, board runner, benchmarks, license headers + benchmarks/ benchmark results per chip +``` + +## Supported operators + +All delegated operators are int8 with per-tensor activations and per-channel +symmetric weights, as annotated by `EspNNQuantizer`. + +| Group | Operators | +|---|---| +| Convolution | `conv2d` (any stride and padding, groups 1 or depthwise, dilation through phase decomposition), `conv_transpose2d`, `linear` | +| Activations | `relu`, `relu6`, `hardtanh`, plus table lookups for `hardswish`, `hardsigmoid`, `sigmoid`, `silu`, `tanh`, `gelu`, `elu`, `leaky_relu` | +| Elementwise | `add`, `mul` (with broadcasting), `softmax` (last dim) | +| Pooling | `max_pool2d`, `avg_pool2d`, `adaptive_avg_pool2d`, `mean` over H and W | +| Resize | `upsample_nearest2d`, `upsample_bilinear2d` (`align_corners` either way) | +| Data movement | `reshape`, `view`, `flatten`, `unflatten`, `permute`, `transpose`, `squeeze`, `unsqueeze`, `slice`, `select`, `expand`, `pad`, `cat`, `stack`, `split`, `chunk`, `unbind` | + +Batch norm is folded into the preceding convolution before quantization. +Partitions can have several inputs and outputs. Anything else stays in the +program and runs on the CPU, including float ops, LSTM, layer norm and +attention matmuls. The partitioner reports what it delegated and what it left +behind. Pass `allow_cpu_fallback=False` to make leftover operators an error. + +## Quantize and lower a model + +[`examples/espressif/quantize_and_lower.py`](../../examples/espressif/quantize_and_lower.py) +contains the whole flow: + +```python +import torch +import torchvision +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +model = torchvision.models.mobilenet_v2(weights="DEFAULT").eval() +example_inputs = (torch.randn(1, 3, 224, 224),) + +# 1. Capture the model and insert observers for the ESP-NN int8 scheme +exported = torch.export.export(model, example_inputs).module() +prepared = prepare_pt2e(exported, EspNNQuantizer()) + +# 2. Calibrate (use real samples from your dataset here) +with torch.no_grad(): + for images in calibration_batches: + prepared(images) +quantized = convert_pt2e(prepared) + +# 3. Lower: int8 regions -> EspNNBackend delegates, the rest -> portable CPU kernels +edge, report = to_espnn_edge(torch.export.export(quantized, example_inputs)) +print(report["partitions"], report["delegated_ops"], report["cpu_ops"]) + +# 4. Serialize the ExecuTorch program +with open("mobilenet_v2_espnn.pte", "wb") as f: + f.write(edge.to_executorch().buffer) +``` + +``` +$ python examples/espressif/quantize_and_lower.py +1 EspNN partition(s), 101 delegated operators, 0 operators left for the CPU +wrote mobilenet_v2_espnn.pte +``` + +### The compiler CLI + +`aot_espnn_compiler.py` builds on the same steps for production use. It +calibrates on an image folder, can run quantization-aware training, and +reports float and int8 accuracy. It also writes a bundle that contains test +vectors, so the board can check its own output: + +```bash +# ExecuTorch program (.pte inside an .xtp bundle) with one epoch of QAT +python examples/espressif/aot_espnn_compiler.py \ + --model mobilenet_v3_small --mode executorch \ + --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val \ + --calib-samples 512 --qat-epochs 1 \ + --output mnv3s.xtp --pte-output mnv3s.pte + +# Standalone int8 graph (.xtb bundle), no ExecuTorch runtime on the device +python examples/espressif/aot_espnn_compiler.py \ + --model mobilenet_v3_small --mode standalone \ + --calib-dir imagenette2-160/train --output mnv3s.xtb + +# Your own model: path/to/file.py:factory, one --input-shape per input +python examples/espressif/aot_espnn_compiler.py \ + --model my_models.py:build_kws --weights none --input-shape 1,1,49,10 \ + --calib-npy kws_calib.npy --output kws.xtp +``` + +Tips: + +- Calibrate on a few hundred real samples. Random data gives useless scales. +- Use per-channel PTQ for MobileNetV2-style networks. Networks with hardswish + and squeeze-excite blocks (MobileNetV3, EfficientNet) lose more accuracy + with PTQ alone, and `--qat-epochs` recovers most of it. +- `--crc-outputs` makes the board compare a CRC of large outputs (such as + segmentation maps) instead of storing them in the bundle. + +## Running on the ESP32-S3 + +You need ESP-IDF v5.1 or newer (tested with v5.5) and an ESP32-S3 board. The +examples assume 16 MB flash and 8 MB octal PSRAM, such as the +ESP32-S3-DevKitC-1 N16R8. Every command below runs in an activated ESP-IDF +shell (`. $IDF_PATH/export.sh`). + +### 1. Cross-compile ExecuTorch + +Skip this step for the standalone path. + +```bash +examples/espressif/scripts/build_executorch_esp.sh esp32s3 +# -> cmake-out-esp-esp32s3/{lib,include} +``` + +The script uses the IDF toolchain file. It builds the ExecuTorch runtime, the +portable kernels and the quantized kernels as static libraries, with exceptions +and RTTI off and with `-mlongcalls`. Set `ET_SELECT_OPS_MODEL=model.pte` to +build only the kernels that a given program needs. + +### 2. Add the component to your application + +`backends/espressif/espnn_runtime` is a regular ESP-IDF component, and it +pulls in `espressif/esp-nn` from the component registry. Add it to your +project with `EXTRA_COMPONENT_DIRS`: + +```cmake +set(EXTRA_COMPONENT_DIRS "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime") +``` + +Then enable the delegate with `CONFIG_ESPNN_EXECUTORCH_DELEGATE=y`, using +menuconfig: *ExecuTorch ESP-NN backend*. The component links +`libexecutorch.a`, the kernel registries and the delegate from +`ESPNN_EXECUTORCH_BUILD_DIR`, which defaults to +`cmake-out-esp-${IDF_TARGET}`. It also registers `EspNNBackend` with the +runtime at startup. + +Other options: + +| Kconfig | Default | Effect | +|---|---|---| +| `ESPNN_PROVIDE_EXECUTORCH_PAL` | y | ExecuTorch platform layer: `esp_timer` clock, `printf` logs, PSRAM-aware allocator | +| `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | +| `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | +| `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | + +Load and run a program with the usual ExecuTorch API. `pte_data` points to +the program, for example embedded in flash with `EMBED_FILES` or read from a +partition: + +```cpp +#include +#include +#include + +using namespace executorch::runtime; + +runtime_init(); +executorch::extension::BufferDataLoader loader(pte_data, pte_size); +auto program = Program::load(&loader).get(); +auto meta = program.method_meta("forward").get(); +/* MemoryManager with the method's planned buffers (PSRAM on the S3), then: */ +auto method = program.load_method("forward", &memory_manager).get(); +method.set_input(input_tensor, 0); +method.execute(); +auto stats = executorch::backends::espressif::delegate_stats(); /* time spent inside ESP-NN delegates */ +``` + +[`examples/espressif/espnn_runner/main/pte_suite.cpp`](../../examples/espressif/espnn_runner/main/pte_suite.cpp) +is a complete working example, including memory planning and output checks. + +### 3. Build, flash and verify on the board + +The runner example builds and flashes a bundle, then compares the board's +output against the host reference: + +```bash +python examples/espressif/scripts/run_bundle.py --port /dev/ttyACM0 \ + --executorch-build-dir cmake-out-esp-esp32s3 mnv3s.xtp +``` + +``` +ESP32-S3 (QFN56) (revision v0.2): 16 MB flash; building executorch runner for esp32s3 +CHIP target=esp32s3 cores=2 revision=2 cpu_mhz=240 flash_mb=16 psram_kb=8192 ... +ET CASE 1/1 name=mobilenet_v3_small ... delegates=1 delegate_us=... cpu_us=... exact=1 status=PASS +ESPNN RUNNER: PASS +RESULT: PASS (1 cases) +``` + +You can also work in the example directly with `idf.py`. See +[`examples/espressif/espnn_runner/README.md`](../../examples/espressif/espnn_runner/README.md). +[`examples/espressif/camera_stream`](../../examples/espressif/camera_stream/README.md) +runs a model live on camera frames and streams the results to a browser. + +### The original ESP32 + +The same flow works on the ESP32 (`idf.py set-target esp32`, or let the +scripts detect the chip). ESP-NN has no SIMD kernels for it and uses portable +C, about 13 million int8 multiply-accumulates per second at 240 MHz, roughly +an order of magnitude slower than the S3. Most ESP32 modules have no PSRAM, so +activations must fit in about 280 KB of internal RAM. Small models fit, such +as keyword spotting, tiny CNNs, and MobileNets at low resolution. Models that +need more RAM fail with a clear allocation error. + +## Benchmarks + +`examples/espressif/scripts/benchmark.py` compiles a fixed set of models. It +groups as many as fit into each flash image, runs them on the connected board +and writes `examples/espressif/benchmarks/.{md,json}`. Every row is +verified against the host reference before it is timed, so a row without +`PASS` is not reported as a result. + +```bash +python examples/espressif/scripts/benchmark.py --port /dev/ttyACM0 \ + --images imagenette2-160 --executorch-build-dir cmake-out-esp-esp32s3 --dashboard +``` + +The results below are at 240 MHz, measured over 10 runs after a warm-up run, +with batch 1. The S3 board was an ESP32-S3 N16R8 (octal PSRAM at 80 MHz, 64 KB +data cache). The ESP32 board was an ESP32-D0WDQ6 (revision 1.0) with 4 MB flash and no PSRAM. + +### ESP32-S3 + +| Model | Input | Path | Ops | Weights | Arena | Latency | On-board check | +|---|---|---|---|---|---|---|---| +| Keyword spotting DS-CNN | 1×1×49×10 | standalone | 12 | 0.03 MB | 16 KB | 16.4 ms | bit-exact | +| Small residual CNN | 1×3×64×64 | standalone | 16 | 0.24 MB | 36 KB | 40.3 ms | bit-exact | +| MobileNetV3-Small | 1×3×96×96 | standalone | 108 | 2.68 MB | 72 KB | 116 ms | bit-exact | +| MobileNetV3-Small | 1×3×224×224 | standalone | 108 | 2.68 MB | 392 KB | 582 ms | bit-exact | +| MobileNetV2 | 1×3×224×224 | standalone | 65 | 3.70 MB | 1.4 MB | 1.61 s | bit-exact | +| MobileNetV3-Large | 1×3×224×224 | standalone | 121 | 5.71 MB | 980 KB | 1.58 s | bit-exact | +| EfficientNet-B0 | 1×3×224×224 | standalone | 190 | 5.65 MB | 2.3 MB | 2.50 s | bit-exact | +| LR-ASPP MobileNetV3 segmentation | 1×3×224×224 | ExecuTorch, 146 delegated / 0 CPU | | 3.43 MB `.pte` | 1.0 MB | 3.58 s | bit-exact (CRC) | +| SSDLite320 MobileNetV3 detection heads | 1×3×320×320 | ExecuTorch, 243 delegated / 0 CPU | | 3.72 MB `.pte` | 2.0 MB | 3.83 s | bit-exact (CRC) | +| Small vision transformer | 1×3×64×64 | ExecuTorch, 11 delegated / 30 CPU | | 0.04 MB `.pte` | 38 KB | 85 ms (delegate 22, CPU 63) | 39.3 dB vs host | + +Arenas that fit internal RAM (the small models) stay there; every 224×224 +model's arena is in PSRAM. The transformer is checked by SQNR because its float CPU +operators (LayerNorm, attention, softmax) round differently from the host. + +### ESP32 (no PSRAM, 4 MB flash) + +Standalone path only. The ExecuTorch path needs ExecuTorch cross-built for +this chip (`build_executorch_esp.sh esp32`), which was not part of this run. + +| Model | Input | Ops | Weights | Arena | Latency | On-board check | +|---|---|---|---|---|---|---| +| Keyword spotting DS-CNN (2.7 MMAC) | 1×1×49×10 | 12 | 0.03 MB | 16 KB | 203.4 ms | bit-exact | +| Small residual CNN (9.3 MMAC) | 1×3×64×64 | 16 | 0.24 MB | 36 KB | 705.0 ms | bit-exact | +| MobileNetV3-Small | 1×3×96×96 | 108 | 2.68 MB | 72 KB | 1440.7 ms | bit-exact | +| MobileNetV3-Small | 1×3×224×224 | 108 | 2.68 MB | 392 KB | does not fit 280 KB RAM | – | +| MobileNetV2 / V3-Large / EfficientNet-B0 at 224 | | | 3.7–5.7 MB | | do not fit 4 MB flash | – | + +Operator suite (`op_suite.xtb`, 20 runs each), all 11 cases bit-exact against +the host reference and ESP-NN's ANSI C kernels: + +| Case | Latency | +|---|---| +| SE block: hardswish LUT, mul, softmax | 6.8 ms | +| Detection heads (reshape/permute/cat) | 9.0 ms | +| Segmentation: dilated conv + transposed conv | 12.0 ms | +| Upsample nearest ×2 | 2.6 ms | +| Upsample bilinear ×2 / align_corners | 2.9 / 2.2 ms | +| Broadcast, pad, split, cat | 2.9 ms | +| Two inputs, two outputs | 1.1 ms | +| Transpose, select, stack | 0.4 ms | + +Full tables with arena sizes and host accuracy are in +[`examples/espressif/benchmarks/`](../../examples/espressif/benchmarks/). + +## Testing + +```bash +pytest backends/espressif/test +``` + +The tests export small models for every operator group, lower them, run the +XTS3 graphs through the reference interpreter and compare the result against +the quantized PyTorch model. CPU fallback tests run the lowered program in the +ExecuTorch Python runtime. On hardware, +`examples/espressif/scripts/make_test_bundles.py` packs the same models into +`op_suite.xtb` and `pte_suite.xtp` for the runner, and every case must be +bit-exact on the board. + +The code follows the ExecuTorch coding style and passes the repository's +`lintrunner` checks (flake8, ufmt, clang-format, cppcheck, cmake-format and +the license and include linters): + +```bash +lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" +``` + +## Limitations + +- Only int8 is delegated. Float32 operators run on ExecuTorch's portable + kernels, which are slow on these chips. +- Activations are per-tensor and weights are per-channel symmetric. Dynamic + shapes are not supported, so each program has a fixed input shape. +- Attention matmuls, LSTM/GRU and normalization layers other than folded batch + norm fall back to the CPU. +- Dual-core execution only helps large convolutions, and it is off by default. +- The delegate handle comes from ExecuTorch's runtime allocator, but each + delegate allocates its activation arena, scratch buffers and optional PSRAM + weight copy with ESP-IDF's `heap_caps` allocator at `init()`, so it can + choose between internal RAM and PSRAM. Nothing is allocated during + `execute()`. + +## License + +Every source file in this backend and its examples is released under the +BSD 3-Clause license (`SPDX-License-Identifier: BSD-3-Clause`), the same +license as ExecuTorch. See the `LICENSE` file at the repository root. ESP-NN +is Apache-2.0 and is fetched from the Espressif component registry, not +vendored. diff --git a/backends/espressif/__init__.py b/backends/espressif/__init__.py new file mode 100644 index 00000000000..ecddace1659 --- /dev/null +++ b/backends/espressif/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""ESP-NN backend for Espressif ESP32 chips (ESP32-S3, ESP32, ESP32-P4, ...).""" + +from .espnn_preprocess import EspNNBackend +from .lowering import to_espnn_edge +from .partition import EspNNPartitioner +from .quantizer import EspNNQuantizer + +__all__ = ["EspNNBackend", "EspNNPartitioner", "EspNNQuantizer", "to_espnn_edge"] diff --git a/backends/espressif/_passes/__init__.py b/backends/espressif/_passes/__init__.py new file mode 100644 index 00000000000..beda70b1315 --- /dev/null +++ b/backends/espressif/_passes/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .fold_batch_norm import fold_batch_norm +from .remove_dropout import remove_dropout + +__all__ = ["fold_batch_norm", "remove_dropout"] diff --git a/backends/espressif/_passes/fold_batch_norm.py b/backends/espressif/_passes/fold_batch_norm.py new file mode 100644 index 00000000000..f3251ae2e87 --- /dev/null +++ b/backends/espressif/_passes/fold_batch_norm.py @@ -0,0 +1,79 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from collections import OrderedDict +from copy import deepcopy + +import torch.nn as nn +from torch.nn.utils.fusion import fuse_conv_bn_eval + + +def _fuse_sequential(module: nn.Sequential) -> tuple[nn.Sequential, int]: + """Fold adjacent Conv2d -> BatchNorm2d pairs inside one nn.Sequential.""" + children = list(module.named_children()) + fused = OrderedDict() + count = 0 + index = 0 + while index < len(children): + name, child = children[index] + if ( + isinstance(child, nn.Conv2d) + and index + 1 < len(children) + and isinstance(children[index + 1][1], nn.BatchNorm2d) + ): + bn_name, batch_norm = children[index + 1] + if child.training or batch_norm.training: + raise RuntimeError( + f"{name} / {bn_name} must be in eval() mode to fold batch norm" + ) + fused[name] = fuse_conv_bn_eval(child, batch_norm) + count += 1 + index += 2 + continue + fused[name] = child + index += 1 + return nn.Sequential(fused), count + + +def _fuse_recursive(module: nn.Module) -> int: + total = 0 + for name, child in list(module.named_children()): + if isinstance(child, nn.Sequential): + fused_child, count = _fuse_sequential(child) + setattr(module, name, fused_child) + total += count + _fuse_recursive(fused_child) + else: + total += _fuse_recursive(child) + return total + + +def fold_batch_norm( + model: nn.Module, *, inplace: bool = False +) -> tuple[nn.Module, int]: + """ + Fold inference-time Conv2d -> BatchNorm2d pairs held in nn.Sequential + containers (the torchvision convention) before export, so that the + quantizer sees plain convolutions. Returns (model, number_of_folds). + """ + if model.training: + raise ValueError("fold_batch_norm() requires model.eval()") + if not inplace: + model = deepcopy(model) + return model, _fuse_recursive(model) + + +def count_conv_bn(model: nn.Module) -> tuple[int, int]: + """Return (number of Conv2d modules, number of BatchNorm2d modules).""" + convs = sum(isinstance(m, nn.Conv2d) for m in model.modules()) + norms = sum(isinstance(m, nn.BatchNorm2d) for m in model.modules()) + return convs, norms + + +__all__ = ["count_conv_bn", "fold_batch_norm"] diff --git a/backends/espressif/_passes/remove_dropout.py b/backends/espressif/_passes/remove_dropout.py new file mode 100644 index 00000000000..60c0e40a3d3 --- /dev/null +++ b/backends/espressif/_passes/remove_dropout.py @@ -0,0 +1,45 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import torch.nn as nn + + +def remove_dropout(model: nn.Module) -> nn.Module: + """ + Replace every nn.Dropout module with nn.Identity(). + + The top-level model must be in eval() mode. + """ + + if model.training: + raise ValueError("remove_dropout() requires model.eval()") + + for name, child in list(model.named_children()): + if isinstance(child, nn.Dropout): + setattr(model, name, nn.Identity()) + else: + remove_dropout_from_children(child) + + return model + + +def remove_dropout_from_children(module: nn.Module) -> None: + """ + Recursively replace Dropout modules. + + Deliberately does NOT check module.training. + The top-level model has already been checked. + """ + + for name, child in list(module.named_children()): + if isinstance(child, nn.Dropout): + setattr(module, name, nn.Identity()) + else: + remove_dropout_from_children(child) diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py new file mode 100644 index 00000000000..184d9a422d1 --- /dev/null +++ b/backends/espressif/espnn_preprocess.py @@ -0,0 +1,58 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import List + +from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult +from executorch.exir.backend.compile_spec_schema import CompileSpec + +from .serialization.xts3 import build_graph, serialize_graph + +logger = logging.getLogger(__name__) + +BACKEND_ID = "EspNNBackend" +INT8_FORMAT_SPEC = CompileSpec("espnn_format", b"int8") +DUMP_ENV = "ESPNN_DELEGATE_OUTPUT" + + +class EspNNBackend(BackendDetails): + """ + Ahead-of-time half of the ESP-NN delegate. + + Each partition produced by EspNNPartitioner is lowered to an XTS3 int8 + graph: ESP-NN operator records, per-channel requantization parameters, + 16-byte aligned int8/int32 constants and an input/output table. The + on-device EspNNBackend (backends/espressif/runtime) parses the blob and + executes it with ESP-NN kernels. + + Set ESPNN_DELEGATE_OUTPUT= to also write the last blob to a file. + """ + + @staticmethod + def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResult: + graph = build_graph(edge_program) + blob = serialize_graph(graph) + logger.info( + "EspNN delegate: %d tensors, %d operations, %d constant bytes, %d byte blob", + len(graph.tensors), + len(graph.operations), + len(graph.constant_data), + len(blob), + ) + output_path = os.environ.get(DUMP_ENV) + if output_path: + Path(output_path).write_bytes(blob) + return PreprocessResult(processed_bytes=blob, debug_handle_map={}) + + +__all__ = ["BACKEND_ID", "EspNNBackend", "INT8_FORMAT_SPEC"] diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt new file mode 100644 index 00000000000..c14c90004df --- /dev/null +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -0,0 +1,96 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +set(espnn_sources "src/xts3_graph.cpp" "src/int8_interpreter.cpp") +if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) + list(APPEND espnn_sources "src/espnn_backend.cpp") + if(CONFIG_ESPNN_PROVIDE_EXECUTORCH_PAL) + list(APPEND espnn_sources "src/espnn_pal.cpp") + endif() +endif() + +idf_component_register( + SRCS + ${espnn_sources} + INCLUDE_DIRS + "include" + REQUIRES + esp_timer + heap + freertos + PRIV_REQUIRES + esp-nn + WHOLE_ARCHIVE +) + +if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) + get_filename_component( + default_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE + ) + set(ESPNN_EXECUTORCH_SOURCE_DIR + "${default_root}" + CACHE PATH "ExecuTorch source tree" + ) + set(executorch_root "${ESPNN_EXECUTORCH_SOURCE_DIR}") + set(ESPNN_EXECUTORCH_BUILD_DIR + "${executorch_root}/cmake-out-esp-${IDF_TARGET}" + CACHE + PATH + "ExecuTorch cross-compiled for this target (examples/espressif/scripts/build_executorch_esp.sh)" + ) + set(et_lib "${ESPNN_EXECUTORCH_BUILD_DIR}/lib") + foreach(required libexecutorch.a libexecutorch_core.a libportable_kernels.a + libkernels_util_all_deps.a + ) + if(NOT EXISTS "${et_lib}/${required}") + message( + FATAL_ERROR + "Missing ${et_lib}/${required}. Build ExecuTorch for ${IDF_TARGET} with " + "examples/espressif/scripts/build_executorch_esp.sh or set " + "-DESPNN_EXECUTORCH_BUILD_DIR=" + ) + endif() + endforeach() + set(et_registries "") + foreach(registry libportable_ops_lib.a libexecutorch_selected_kernels.a + libquantized_ops_lib.a + ) + if(EXISTS "${et_lib}/${registry}") + list(APPEND et_registries "${et_lib}/${registry}") + endif() + endforeach() + set(et_kernels "${et_lib}/libportable_kernels.a") + if(EXISTS "${et_lib}/libquantized_kernels.a") + list(APPEND et_kernels "${et_lib}/libquantized_kernels.a") + endif() + target_include_directories( + ${COMPONENT_LIB} + PUBLIC + "${ESPNN_EXECUTORCH_BUILD_DIR}/include" + "${ESPNN_EXECUTORCH_BUILD_DIR}/include/executorch/runtime/core/portable_type/c10" + "${executorch_root}/.." + ) + target_compile_definitions( + ${COMPONENT_LIB} PUBLIC C10_USING_CUSTOM_GENERATED_MACROS + ) + # Kernel and backend registration happens in static initializers, so these + # archives are linked whole. One flag per archive: CMake deduplicates a bare + # -Wl,--whole-archive pair. + set(et_whole_archives "") + foreach(archive ${et_registries} "${et_lib}/libexecutorch.a") + list(APPEND et_whole_archives + "-Wl,--whole-archive,${archive},--no-whole-archive" + ) + endforeach() + target_link_libraries( + ${COMPONENT_LIB} + INTERFACE ${et_whole_archives} ${et_kernels} + "${et_lib}/libkernels_util_all_deps.a" + "${et_lib}/libexecutorch_core.a" + ) +endif() diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig new file mode 100644 index 00000000000..28152ff3760 --- /dev/null +++ b/backends/espressif/espnn_runtime/Kconfig @@ -0,0 +1,39 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +menu "ExecuTorch ESP-NN backend" + + config ESPNN_EXECUTORCH_DELEGATE + bool "Build the ExecuTorch delegate (EspNNBackend) and link the ExecuTorch runtime" + default n + help + Needs ExecuTorch cross-compiled for this chip, see + examples/espressif/scripts/build_executorch_esp.sh. Without it the + component still provides the standalone XTS3 int8 interpreter. + + config ESPNN_PROVIDE_EXECUTORCH_PAL + bool "Provide the ExecuTorch platform layer (esp_timer clock, printf logs, PSRAM allocator)" + default y + depends on ESPNN_EXECUTORCH_DELEGATE + + config ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + bool "Copy delegate weights from flash into PSRAM when they fit" + default y + depends on ESPNN_EXECUTORCH_DELEGATE && SPIRAM + + config ESPNN_DELEGATE_DUAL_CORE + bool "Split ESP-NN kernels across both cores inside delegates" + default n + depends on ESPNN_EXECUTORCH_DELEGATE + + config ESPNN_DELEGATE_PROFILE + bool "Print a per-operator profile after each delegate's first run" + default n + depends on ESPNN_EXECUTORCH_DELEGATE + +endmenu diff --git a/backends/espressif/espnn_runtime/idf_component.yml b/backends/espressif/espnn_runtime/idf_component.yml new file mode 100644 index 00000000000..2891ae084a6 --- /dev/null +++ b/backends/espressif/espnn_runtime/idf_component.yml @@ -0,0 +1,13 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +description: ExecuTorch ESP-NN backend runtime (XTS3 int8 interpreter and EspNNBackend delegate) +dependencies: + idf: + version: ">=5.1" + espressif/esp-nn: "^1.4.0" diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h new file mode 100644 index 00000000000..f6047d07602 --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { + +/// Backend ID that `EspNNBackend.preprocess()` writes into lowered programs. +constexpr const char* kEspNNBackendId = "EspNNBackend"; + +/** + * Counters accumulated by every EspNNBackend delegate since startup or the + * last reset_delegate_stats() call. + */ +struct DelegateStats { + /// Number of delegate instances initialized. + uint32_t delegates; + /// Number of delegate execute() calls. + uint32_t calls; + /// Total time spent in execute(), including input and output conversion. + int64_t total_us; + /// Sum of the activation arenas held by the initialized delegates. + size_t arena_bytes; +}; + +/** + * Registers EspNNBackend with the ExecuTorch runtime. + * + * Registration also happens from a static initializer when the component is + * linked whole-archive; calling this is only needed when the linker could + * drop that initializer. Safe to call more than once. + * + * @retval true The backend is registered. + */ +bool register_espnn_backend(); + +/// Returns a snapshot of the delegate counters. +DelegateStats delegate_stats(); + +/// Zeroes all delegate counters. +void reset_delegate_stats(); + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h new file mode 100644 index 00000000000..11bcc22d9f5 --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -0,0 +1,235 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace backends { +namespace espressif { + +/// Which ESP-NN kernel family runs the graph. +enum class KernelMode : uint8_t { + /// Chip-optimized ESP-NN kernels (SIMD on the ESP32-S3). + OPTIMIZED = 0, + /// ESP-NN's portable ANSI C kernels, used to cross-check OPTIMIZED. + REFERENCE = 1, +}; + +/// Options for Int8Interpreter::load(). +struct Int8RuntimeOptions { + /// Kernel family used by invoke(). + KernelMode kernel_mode = KernelMode::OPTIMIZED; + /// Copy weights from memory-mapped flash into PSRAM when they fit. + bool copy_constants_to_psram = false; + /// Place the activation arena in internal RAM when it fits, else PSRAM. + bool prefer_internal_arena = true; + /// Split large convolutions across both cores. + bool dual_core = false; + /// Internal RAM left free when deciding whether the arena fits there. + size_t internal_reserve_bytes = 48 * 1024; +}; + +/** + * Shape and quantization of a graph input or output. Rank-4 tensors are + * described in NCHW terms but stored on the device as NHWC. + */ +struct Int8TensorInfo { + uint32_t tensor_id; + uint8_t rank; + int32_t shape[4]; + int32_t height; + int32_t width; + int32_t channels; + size_t bytes; + float scale; + int32_t zero_point; +}; + +/** + * Runs an XTS3 int8 graph (the EspNNBackend delegate payload) with ESP-NN. + * + * The interpreter does not copy the graph: the blob passed to load() must + * stay valid until reset() or destruction. Activations live in one arena + * planned at load time, so invoke() does not allocate. + */ +class Int8Interpreter { + public: + Int8Interpreter() = default; + ~Int8Interpreter(); + Int8Interpreter(const Int8Interpreter&) = delete; + Int8Interpreter& operator=(const Int8Interpreter&) = delete; + + /** + * Validates the graph, plans activation memory and allocates the arena, + * scratch buffers and requantization tables. + * + * @param[in] blob XTS3 graph, kept by reference. + * @param[in] size Size of @p blob in bytes. + * @param[in] options Placement and kernel options. + * @retval true The graph is ready for invoke(). + * @retval false The graph is malformed or memory could not be allocated; + * the reason is printed. + */ + bool load( + const uint8_t* blob, + size_t size, + const Int8RuntimeOptions& options = {}); + + /// Frees everything allocated by load(). The interpreter can be reused. + void reset(); + + /** + * Runs every operation once on the current input data. + * + * @retval false The graph is not loaded or an operation failed. + */ + bool invoke(); + + /// Number of graph inputs. + size_t input_count() const { + return input_count_; + } + /// Number of graph outputs. + size_t output_count() const { + return output_count_; + } + /// Describes input @p index. @retval false @p index is out of range. + bool input_info(size_t index, Int8TensorInfo& info) const; + /// Describes output @p index. @retval false @p index is out of range. + bool output_info(size_t index, Int8TensorInfo& info) const; + /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. + int8_t* input_data(size_t index); + /// Device-layout (NHWC) int8 buffer of output @p index, or nullptr. + const int8_t* output_data(size_t index) const; + /// Raw data of any graph tensor, constant or activation, or nullptr. + const int8_t* tensor_data(uint32_t tensor_id) const; + + /** + * Quantizes float NCHW values into input @p index. + * + * @param[in] count Number of values; must equal the input's size. + */ + bool quantize_input_nchw(size_t index, const float* values, size_t count); + + /** + * Normalizes interleaved 8-bit RGB pixels with (x / 255 - mean) / stddev + * and quantizes them into a 3-channel input of the same width and height. + */ + bool set_input_rgb888( + size_t index, + const uint8_t* rgb, + int32_t width, + int32_t height, + const float mean[3], + const float stddev[3]); + + /// Dequantizes output @p index in device layout into @p values. + bool dequantize_output(size_t index, float* values, size_t count) const; + + /// Index of the largest int8 value of output @p index, or -1. + int32_t output_argmax(size_t index) const; + + /// Switches kernels for later invoke() calls without reloading. + void set_kernel_mode(KernelMode mode) { + options_.kernel_mode = mode; + } + KernelMode kernel_mode() const { + return options_.kernel_mode; + } + const GraphView& graph() const { + return graph_; + } + /// Duration of the last invoke() in microseconds. + int64_t last_invoke_us() const { + return last_invoke_us_; + } + /// Duration of operation @p op during the last invoke(), or 0. + int64_t op_time_us(uint32_t op) const; + /// Size of the activation arena. + size_t arena_bytes() const { + return arena_size_; + } + /// Whether the arena was placed in internal RAM rather than PSRAM. + bool arena_in_internal_ram() const { + return arena_internal_; + } + /// Size of the shared ESP-NN scratch buffer. + size_t scratch_bytes() const { + return scratch_size_; + } + /// Size of the precomputed requantization tables. + size_t parameter_bytes() const { + return params_size_; + } + /// Whether weights were copied into PSRAM by load(). + bool constants_copied() const { + return blob_copy_ != nullptr; + } + /// Prints tensor, operation and memory statistics. + void print_summary() const; + /// Prints per-operation times of the last invoke(). + void print_profile() const; + + private: + struct OpState; + + bool plan_memory(); + bool prepare_operations(); + void rebase_constants(const uint8_t* old_base, size_t size); + bool tensor_info(uint32_t tensor_id, Int8TensorInfo& info) const; + int8_t* activation(uint32_t tensor_id) const; + const int8_t* operand(uint32_t tensor_id) const; + void run_generic_conv(const OpState& s, const int8_t* input, int8_t* output) + const; + void run_dilated_by_phases( + const OpState& s, + const int8_t* input, + int8_t* output) const; + void run_copy(const OpState& s, int8_t* output) const; + void run_upsample(const OpState& s, const int8_t* input, int8_t* output) + const; + void run_transposed_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const; + bool run_operation(uint32_t index); + + Int8RuntimeOptions options_{}; + GraphView graph_{}; + bool loaded_ = false; + uint8_t* blob_copy_ = nullptr; + TensorRecord* tensors_ = nullptr; + OperationRecord* operations_ = nullptr; + OpState* op_states_ = nullptr; + size_t* tensor_offsets_ = nullptr; + uint8_t* arena_ = nullptr; + size_t arena_size_ = 0; + bool arena_internal_ = false; + void* scratch_ = nullptr; + size_t scratch_size_ = 0; + int8_t* phase_buffer_ = nullptr; + size_t phase_bytes_ = 0; + int32_t* params_ = nullptr; + size_t params_size_ = 0; + uint32_t inputs_[MAX_GRAPH_IO]{}; + uint32_t outputs_[MAX_GRAPH_IO]{}; + size_t input_count_ = 0; + size_t output_count_ = 0; + int64_t last_invoke_us_ = 0; +}; + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h new file mode 100644 index 00000000000..cc74285f91c --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -0,0 +1,240 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +/** + * @file + * Parser for XTS3, the int8 graph format produced by EspNNBackend.preprocess() + * (backends/espressif/serialization/xts3.py). All multi-byte fields are + * little-endian. A graph is a header, an optional input/output table + * (version 6), fixed-size tensor and operation records, a parameter region + * and a constant region. parse_graph() bounds-checks every region and record + * before any accessor reads it. + */ + +namespace executorch { +namespace backends { +namespace espressif { + +/// Graph versions accepted by this runtime; 6 adds the input/output table. +static constexpr uint32_t GRAPH_ESP_NN_INT8_VERSION = 5; +static constexpr uint32_t GRAPH_ESP_NN_INT8_IO_VERSION = 6; +static constexpr size_t GRAPH_HEADER_SIZE = 22; +static constexpr size_t GRAPH_IO_TABLE_HEADER_SIZE = 4; +static constexpr size_t MAX_GRAPH_IO = 16; +static constexpr size_t TENSOR_RECORD_SIZE = 36; +static constexpr size_t OPERATION_RECORD_SIZE = 44; +static constexpr size_t MAX_GRAPH_TENSORS = 1024; +static constexpr size_t MAX_GRAPH_OPERATIONS = 1024; + +/// TensorRecord::flags bits. +static constexpr uint8_t TENSOR_FLAG_CONSTANT = 1U << 0; +static constexpr uint8_t TENSOR_FLAG_QUANTIZED = 1U << 1; +static constexpr uint8_t TENSOR_FLAG_GRAPH_INPUT = 1U << 2; +static constexpr uint8_t TENSOR_FLAG_GRAPH_OUTPUT = 1U << 3; + +/// Serialized parameter block sizes and modes, mirrored in xts3.py. +static constexpr size_t INT8_CONV_BASE_PARAMS_SIZE = 20; +static constexpr size_t INT8_ADD_PARAMS_SIZE = 36; +static constexpr size_t INT8_PAIR_PARAMS_SIZE = 8; +static constexpr size_t INT8_POOL_PARAMS_SIZE = 16; +static constexpr size_t INT8_LUT_PARAMS_SIZE = 256; +static constexpr size_t INT8_MUL_PARAMS_SIZE = 20; +static constexpr size_t INT8_SOFTMAX_PARAMS_SIZE = 1024; +static constexpr int32_t MUL_MODE_ELEMENTWISE = 0; +static constexpr int32_t MUL_MODE_CHANNEL_BROADCAST = 1; +static constexpr int32_t SOFTMAX_OUTPUT_ZERO_POINT = -128; +static constexpr size_t INT8_PARAMETER_ALIGNMENT = 16; +static constexpr size_t COPY_HEADER_SIZE = 12; +static constexpr size_t COPY_MAX_RANK = 6; +static constexpr size_t COPY_SEGMENT_SIZE = 108; +static constexpr int32_t COPY_FLAG_FILL = 1; +static constexpr size_t UPSAMPLE_HEADER_SIZE = 20; +static constexpr int32_t UPSAMPLE_MODE_NEAREST = 0; +static constexpr int32_t UPSAMPLE_MODE_BILINEAR = 1; +static constexpr int32_t UPSAMPLE_WEIGHT_BITS = 10; +static constexpr size_t TRANSPOSE_CONV_PARAMS_SIZE = 52; + +/// Operation types; values are part of the file format. +enum class OpCode : uint8_t { + CONV2D = 0, + DEPTHWISE_CONV2D = 1, + RELU6 = 2, + ADD = 3, + AVGPOOL2D = 4, + LINEAR = 5, + FLATTEN = 6, + MAX_POOL2D = 7, + AVG_POOL2D = 8, + CONCAT = 9, + LUT = 10, + MUL = 11, + SOFTMAX = 12, + COPY = 13, + UPSAMPLE = 14, + TRANSPOSE_CONV2D = 15, +}; + +static constexpr uint8_t OPCODE_COUNT = 16; + +/// Tensor element types; delegated graphs only use INT8 and INT32. +enum class DataType : uint8_t { + UNKNOWN = 0, + FLOAT32 = 1, + INT8 = 2, + UINT8 = 3, + INT32 = 4, +}; + +/// Fixed graph header. +struct GraphHeader { + char magic[4]; + uint16_t version; + uint32_t num_tensors; + uint32_t num_operations; + uint32_t parameter_size; + uint32_t constant_size; +}; + +/// One tensor; shape is NCHW for rank 4, stored on the device as NHWC. +struct TensorRecord { + uint32_t tensor_id; + uint32_t constant_offset; + uint8_t dtype; + uint8_t flags; + uint8_t rank; + int32_t shape[4]; + float quant_scale; + int32_t quant_zero_point; +}; + +/// One operation with its tensor IDs and parameter block. +struct OperationRecord { + uint8_t op_id; + uint8_t opcode; + uint8_t num_inputs; + uint8_t num_outputs; + uint32_t inputs[6]; + uint32_t outputs[2]; + uint32_t parameter_offset; + uint32_t parameter_size; +}; + +/// Validated view of a graph; points into the caller's buffer. +struct GraphView { + const uint8_t* data; + size_t size; + size_t tensor_record_size; + GraphHeader header; + size_t tensor_table_offset; + size_t operation_table_offset; + size_t parameter_offset; + size_t constant_offset; + const uint8_t* parameter_data; + const uint8_t* constant_data; + uint16_t num_inputs; + uint16_t num_outputs; + const uint8_t* io_ids; +}; + +/// Decoded convolution parameters; multipliers and shifts are per channel. +struct Int8ConvParams { + int32_t kernel_h; + int32_t kernel_w; + int32_t stride_h; + int32_t stride_w; + int32_t pad_h; + int32_t pad_w; + int32_t groups; + int32_t activation_min; + int32_t activation_max; + const uint8_t* multipliers; + const uint8_t* shifts; + int32_t dilation_h; + int32_t dilation_w; +}; + +/// Decoded requantization parameters of an elementwise add. +struct Int8AddParams { + int32_t input1_mult; + int32_t input1_shift; + int32_t input2_mult; + int32_t input2_shift; + int32_t left_shift; + int32_t output_mult; + int32_t output_shift; + int32_t activation_min; + int32_t activation_max; +}; + +/// Reads a little-endian 32-bit value from unaligned memory. +uint32_t read_le_u32(const uint8_t* p); +/// Reads a little-endian 32-bit signed value from unaligned memory. +int32_t read_le_i32(const uint8_t* p); + +/** + * Validates a serialized graph and fills @p graph. + * + * Checks the magic, version, region bounds, every tensor and operation + * record, and each operation's parameter block against its tensor shapes. + * + * @param[in] data Serialized graph; must outlive @p graph. + * @param[in] size Size of @p data in bytes. + * @param[out] graph View used by the other functions. + * @retval false The graph is malformed; the reason is printed. + */ +bool parse_graph(const uint8_t* data, size_t size, GraphView& graph); +/// Whether @p version is an int8 graph version this runtime can execute. +bool is_int8_graph_version(uint32_t version); +/// Number of graph inputs. +uint32_t graph_input_count(const GraphView& graph); +/// Number of graph outputs. +uint32_t graph_output_count(const GraphView& graph); +/// Tensor ID of input @p index, or UINT32_MAX if out of range. +uint32_t graph_input_id(const GraphView& graph, uint32_t index); +/// Tensor ID of output @p index, or UINT32_MAX if out of range. +uint32_t graph_output_id(const GraphView& graph, uint32_t index); +/// Decodes tensor record @p index. @retval false Out of range. +bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor); +/// Decodes operation record @p index. @retval false Out of range. +bool read_operation( + const GraphView& graph, + uint32_t index, + OperationRecord& operation); +/// Decodes a convolution's parameter block for @p channels output channels. +bool read_int8_conv_params( + const GraphView& graph, + const OperationRecord& op, + uint32_t channels, + Int8ConvParams& params); +/// Decodes an elementwise add's parameter block. +bool read_int8_add_params( + const GraphView& graph, + const OperationRecord& op, + Int8AddParams& params); +/// Number of elements described by the tensor's shape. +size_t tensor_element_count(const TensorRecord& tensor); +/// Size of the tensor's data in bytes. +size_t tensor_data_size(const TensorRecord& tensor); +/// Constant data of @p tensor; only meaningful for TENSOR_FLAG_CONSTANT +/// tensors, whose offsets parse_graph() has checked. +const uint8_t* tensor_constant_data( + const GraphView& graph, + const TensorRecord& tensor); +/// Printable operation name. +const char* opcode_name(OpCode opcode); + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp new file mode 100644 index 00000000000..4e542706c5e --- /dev/null +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -0,0 +1,281 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +using executorch::aten::ScalarType; +using executorch::aten::Tensor; +using executorch::runtime::ArrayRef; +using executorch::runtime::Backend; +using executorch::runtime::BackendExecutionContext; +using executorch::runtime::BackendInitContext; +using executorch::runtime::BackendInterface; +using executorch::runtime::CompileSpec; +using executorch::runtime::DelegateHandle; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::Result; +using executorch::runtime::Span; + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +DelegateStats g_stats{}; + +int8_t quantize(float value, float inverse_scale, int32_t zero_point) { + const float scaled = value * inverse_scale; + if (!std::isfinite(scaled)) + return scaled > 0.0f ? 127 : -128; + if (scaled > 1.0e6f) + return 127; + if (scaled < -1.0e6f) + return -128; + int32_t q = static_cast(std::nearbyintf(scaled)) + zero_point; + if (q < -128) + q = -128; + if (q > 127) + q = 127; + return static_cast(q); +} + +bool same_elements(const Tensor& tensor, const Int8TensorInfo& info) { + return static_cast(tensor.numel()) == info.bytes; +} + +bool load_input( + Int8Interpreter& interpreter, + size_t index, + const Tensor& tensor) { + Int8TensorInfo info{}; + if (!interpreter.input_info(index, info) || !same_elements(tensor, info)) + return false; + int8_t* destination = interpreter.input_data(index); + const size_t plane = static_cast(info.height) * info.width; + const size_t channels = static_cast(info.channels); + const bool nchw = info.rank == 4; + if (tensor.scalar_type() == ScalarType::Float) { + const float* source = tensor.const_data_ptr(); + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + if (!nchw) { + for (size_t i = 0; i < info.bytes; ++i) + destination[i] = quantize(source[i], inverse, info.zero_point); + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) { + destination[p * channels + c] = + quantize(source[c * plane + p], inverse, info.zero_point); + } + } + return true; + } + if (tensor.scalar_type() == ScalarType::Char) { + const int8_t* source = tensor.const_data_ptr(); + if (!nchw) { + std::memcpy(destination, source, info.bytes); + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) + destination[p * channels + c] = source[c * plane + p]; + } + return true; + } + return false; +} + +bool store_output( + const Int8Interpreter& interpreter, + size_t index, + Tensor& tensor) { + Int8TensorInfo info{}; + if (!interpreter.output_info(index, info) || !same_elements(tensor, info)) + return false; + const int8_t* source = interpreter.output_data(index); + const size_t plane = static_cast(info.height) * info.width; + const size_t channels = static_cast(info.channels); + const bool nchw = info.rank == 4; + if (tensor.scalar_type() == ScalarType::Float) { + float* destination = tensor.mutable_data_ptr(); + if (!nchw) { + for (size_t i = 0; i < info.bytes; ++i) { + destination[i] = + static_cast(source[i] - info.zero_point) * info.scale; + } + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) { + destination[c * plane + p] = + static_cast(source[p * channels + c] - info.zero_point) * + info.scale; + } + } + return true; + } + if (tensor.scalar_type() == ScalarType::Char) { + int8_t* destination = tensor.mutable_data_ptr(); + if (!nchw) { + std::memcpy(destination, source, info.bytes); + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) + destination[c * plane + p] = source[p * channels + c]; + } + return true; + } + return false; +} + +struct DelegateState { + Int8Interpreter interpreter; + bool profiled = false; +}; + +class EspNNBackendInterface final : public BackendInterface { + public: + bool is_available() const override { + return true; + } + + Result init( + BackendInitContext& context, + FreeableBuffer* processed, + ArrayRef) const override { + void* memory = context.get_runtime_allocator()->allocate( + sizeof(DelegateState), alignof(DelegateState)); + if (memory == nullptr) + return Error::MemoryAllocationFailed; + DelegateState* state = new (memory) DelegateState(); + Int8RuntimeOptions options{}; +#if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + options.copy_constants_to_psram = true; +#endif +#if CONFIG_ESPNN_DELEGATE_DUAL_CORE + options.dual_core = true; +#endif + if (!state->interpreter.load( + static_cast(processed->data()), + processed->size(), + options)) { + state->~DelegateState(); + return Error::DelegateInvalidCompatibility; + } + ++g_stats.delegates; + g_stats.arena_bytes += state->interpreter.arena_bytes(); + return state; + } + + Error execute( + BackendExecutionContext&, + DelegateHandle* handle, + Span args) const override { + DelegateState* state = static_cast(handle); + Int8Interpreter& interpreter = state->interpreter; + const size_t inputs = interpreter.input_count(); + const size_t outputs = interpreter.output_count(); + if (args.size() != inputs + outputs) { + std::printf( + "EspNN delegate: expected %u arguments, got %u\n", + static_cast(inputs + outputs), + static_cast(args.size())); + return Error::InvalidArgument; + } + const int64_t started = esp_timer_get_time(); + for (size_t i = 0; i < inputs; ++i) { + if (!args[i]->isTensor() || + !load_input(interpreter, i, args[i]->toTensor())) { + std::printf( + "EspNN delegate: input %u has an unsupported dtype or size\n", + static_cast(i)); + return Error::InvalidArgument; + } + } + if (!interpreter.invoke()) + return Error::Internal; + for (size_t o = 0; o < outputs; ++o) { + if (!args[inputs + o]->isTensor()) + return Error::InvalidArgument; + Tensor tensor = args[inputs + o]->toTensor(); + if (!store_output(interpreter, o, tensor)) { + std::printf( + "EspNN delegate: output %u has an unsupported dtype or size\n", + static_cast(o)); + return Error::InvalidArgument; + } + } + ++g_stats.calls; + g_stats.total_us += esp_timer_get_time() - started; +#if CONFIG_ESPNN_DELEGATE_PROFILE + if (!state->profiled) { + state->profiled = true; + std::printf( + "EspNN delegate profile (%u ops, %lld us):\n", + static_cast(interpreter.graph().header.num_operations), + static_cast(interpreter.last_invoke_us())); + interpreter.print_profile(); + } +#endif + return Error::Ok; + } + + void destroy(DelegateHandle* handle) const override { + if (handle != nullptr) + static_cast(handle)->~DelegateState(); + } +}; + +EspNNBackendInterface g_backend; +bool g_registered = executorch::runtime::register_backend( + Backend{kEspNNBackendId, &g_backend}) == Error::Ok; + +} // namespace + +bool register_espnn_backend() { + if (!g_registered) { + g_registered = executorch::runtime::register_backend( + Backend{kEspNNBackendId, &g_backend}) == Error::Ok; + } + return g_registered; +} + +DelegateStats delegate_stats() { + return g_stats; +} + +void reset_delegate_stats() { + g_stats = {}; +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/espnn_pal.cpp b/backends/espressif/espnn_runtime/src/espnn_pal.cpp new file mode 100644 index 00000000000..2926893d95a --- /dev/null +++ b/backends/espressif/espnn_runtime/src/espnn_pal.cpp @@ -0,0 +1,59 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include + +#include +#include + +extern "C" { + +void et_pal_init(void) {} + +ET_NORETURN void et_pal_abort(void) { + std::printf("ExecuTorch abort\n"); + std::abort(); +} + +et_timestamp_t et_pal_current_ticks(void) { + return static_cast(esp_timer_get_time()); +} + +et_tick_ratio_t et_pal_ticks_to_ns_multiplier(void) { + return {1000, 1}; +} + +void et_pal_emit_log_message( + et_timestamp_t, + et_pal_log_level_t level, + const char* filename, + const char*, + size_t line, + const char* message, + size_t) { + std::printf( + "ET %c %s:%u %s\n", + static_cast(level), + filename, + static_cast(line), + message); +} + +void* et_pal_allocate(size_t size) { + return heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); +} + +void et_pal_free(void* pointer) { + heap_caps_free(pointer); +} +} diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp new file mode 100644 index 00000000000..c6d1848eee7 --- /dev/null +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -0,0 +1,1767 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +constexpr size_t ARENA_ALIGNMENT = 16; +constexpr int64_t YIELD_INTERVAL_US = 1000000; + +size_t align_up(size_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +void* allocate_preferring_internal( + size_t bytes, + bool prefer_internal, + size_t reserve, + bool* internal) { + if (bytes == 0) + return nullptr; + if (prefer_internal) { + const size_t largest = + heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (largest > bytes + reserve) { + void* memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (memory != nullptr) { + if (internal != nullptr) + *internal = true; + return memory; + } + } + } + void* memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (memory != nullptr) { + if (internal != nullptr) + *internal = false; + return memory; + } + memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (internal != nullptr) + *internal = memory != nullptr; + return memory; +} + +int8_t clamp_to_int8(int32_t value) { + if (value < -128) + return -128; + if (value > 127) + return 127; + return static_cast(value); +} + +int32_t rounding_high_mul(int32_t a, int32_t b) { + if (a == b && a == INT32_MIN) + return INT32_MAX; + int64_t nudge = 1 << 30; + if ((a < 0) ^ (b < 0)) + nudge = 1 - nudge; + const int64_t product = static_cast(a) * b + nudge; + const int64_t to_add = product < 0 ? (1LL << 31) - 1 : 0; + return static_cast((product + to_add) >> 31); +} + +int32_t rounding_shift_right(int32_t value, int32_t exponent) { + const int32_t mask = static_cast((1LL << exponent) - 1); + const int32_t remainder = value & mask; + int32_t result = value >> exponent; + const int32_t threshold = (mask >> 1) + (result < 0 ? 1 : 0); + if (remainder > threshold) + result += 1; + return result; +} + +int32_t requantize(int32_t value, int32_t multiplier, int32_t shift) { + const int32_t left = shift > 0 ? shift : 0; + const int32_t right = shift > 0 ? 0 : -shift; + const int32_t shifted = + static_cast(static_cast(value) << left); + return rounding_shift_right(rounding_high_mul(shifted, multiplier), right); +} + +int8_t quantize_value(float value, float inverse_scale, int32_t zero_point) { + const float scaled = value * inverse_scale; + if (!std::isfinite(scaled)) + return clamp_to_int8(scaled > 0.0f ? 127 : -128); + if (scaled > 1.0e6f) + return 127; + if (scaled < -1.0e6f) + return -128; + return clamp_to_int8( + static_cast(std::nearbyintf(scaled)) + zero_point); +} + +} // namespace + +struct Int8Interpreter::OpState { + OpCode opcode; + uint32_t input0; + uint32_t input1; + uint32_t output; + uint8_t input_count; + uint32_t inputs[6]; + int32_t pool[6]; + const int8_t* weights; + const int32_t* bias; + data_dims_t input_dims; + data_dims_t filter_dims; + data_dims_t output_dims; + conv_params_t conv; + dw_conv_params_t dw; + quant_data_t quant; + int32_t* multipliers; + int32_t* shifts; + Int8AddParams add; + int32_t value0; + int32_t value1; + int32_t input_zero_point; + int32_t output_zero_point; + uint32_t elements; + int32_t rows; + int32_t cols; + bool transpose; + bool generic; + const uint8_t* table; + int32_t dilation_h; + int32_t dilation_w; + int32_t batches; + int64_t total_us; + int64_t last_us; + uint32_t calls; +}; + +Int8Interpreter::~Int8Interpreter() { + reset(); +} + +void Int8Interpreter::reset() { + if (arena_ != nullptr) + heap_caps_free(arena_); + if (scratch_ != nullptr) + heap_caps_free(scratch_); + if (phase_buffer_ != nullptr) + heap_caps_free(phase_buffer_); + phase_buffer_ = nullptr; + phase_bytes_ = 0; + if (params_ != nullptr) + heap_caps_free(params_); + if (blob_copy_ != nullptr) + heap_caps_free(blob_copy_); + if (tensors_ != nullptr) + heap_caps_free(tensors_); + if (operations_ != nullptr) + heap_caps_free(operations_); + if (op_states_ != nullptr) + heap_caps_free(op_states_); + if (tensor_offsets_ != nullptr) + heap_caps_free(tensor_offsets_); + arena_ = nullptr; + scratch_ = nullptr; + params_ = nullptr; + blob_copy_ = nullptr; + tensors_ = nullptr; + operations_ = nullptr; + op_states_ = nullptr; + tensor_offsets_ = nullptr; + arena_size_ = 0; + scratch_size_ = 0; + params_size_ = 0; + input_count_ = 0; + output_count_ = 0; + loaded_ = false; + graph_ = {}; +} + +bool Int8Interpreter::load( + const uint8_t* blob, + size_t size, + const Int8RuntimeOptions& options) { + reset(); + options_ = options; + if (!parse_graph(blob, size, graph_)) { + std::printf("ERROR: int8 runtime rejected the model blob\n"); + return false; + } + if (!is_int8_graph_version(graph_.header.version)) { + std::printf( + "ERROR: model is XTS3 v%u; the int8 runtime requires v%u or v%u\n", + static_cast(graph_.header.version), + static_cast(GRAPH_ESP_NN_INT8_VERSION), + static_cast(GRAPH_ESP_NN_INT8_IO_VERSION)); + return false; + } + const bool misaligned = + (reinterpret_cast(graph_.constant_data) & 15U) != 0; + if (misaligned) { + blob_copy_ = static_cast(allocate_preferring_internal( + size, false, options_.internal_reserve_bytes, nullptr)); + if (blob_copy_ == nullptr) { + std::printf( + "ERROR: unable to copy %u-byte misaligned model into RAM\n", + static_cast(size)); + return false; + } + std::memcpy(blob_copy_, blob, size); + if (!parse_graph(blob_copy_, size, graph_)) + return false; + std::printf( + "WARNING: model constants were not 16-byte aligned; copied to RAM\n"); + } + + const uint32_t tensor_count = graph_.header.num_tensors; + const uint32_t op_count = graph_.header.num_operations; + tensors_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); + operations_ = static_cast( + heap_caps_calloc(op_count, sizeof(OperationRecord), MALLOC_CAP_8BIT)); + op_states_ = static_cast( + heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); + tensor_offsets_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + if (tensors_ == nullptr || operations_ == nullptr || op_states_ == nullptr || + tensor_offsets_ == nullptr) { + std::printf("ERROR: unable to allocate interpreter tables\n"); + reset(); + return false; + } + for (uint32_t i = 0; i < tensor_count; ++i) + read_tensor(graph_, i, tensors_[i]); + input_count_ = graph_input_count(graph_); + output_count_ = graph_output_count(graph_); + if (input_count_ > MAX_GRAPH_IO || output_count_ > MAX_GRAPH_IO) { + std::printf( + "ERROR: model has more than %u inputs or outputs\n", + static_cast(MAX_GRAPH_IO)); + reset(); + return false; + } + for (uint32_t i = 0; i < input_count_; ++i) + inputs_[i] = graph_input_id(graph_, i); + for (uint32_t i = 0; i < output_count_; ++i) + outputs_[i] = graph_output_id(graph_, i); + for (uint32_t i = 0; i < op_count; ++i) + read_operation(graph_, i, operations_[i]); + + if (!plan_memory() || !prepare_operations()) { + reset(); + return false; + } + if (options_.copy_constants_to_psram && blob_copy_ == nullptr && + heap_caps_get_total_size(MALLOC_CAP_SPIRAM) != 0) { + const size_t largest = + heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (largest >= size + 64 * 1024) { + blob_copy_ = static_cast(heap_caps_aligned_alloc( + ARENA_ALIGNMENT, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + } + if (blob_copy_ == nullptr) { + std::printf( + "WARNING: not enough PSRAM to copy %u model bytes; reading weights from flash\n", + static_cast(size)); + } else { + std::memcpy(blob_copy_, graph_.data, size); + const uint8_t* old_base = graph_.data; + if (!parse_graph(blob_copy_, size, graph_)) { + reset(); + return false; + } + rebase_constants(old_base, size); + } + } + if (options_.dual_core) { +#if ESP_NN_DUAL_CORE_SUPPORTED + esp_nn_dual_core_enable(); +#else + std::printf("WARNING: dual-core ESP-NN is not supported on this target\n"); +#endif + } + loaded_ = true; + return true; +} + +bool Int8Interpreter::plan_memory() { + const uint32_t tensor_count = graph_.header.num_tensors; + const int32_t op_count = static_cast(graph_.header.num_operations); + int32_t* first = static_cast( + heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + int32_t* last = static_cast( + heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + size_t* bytes = static_cast( + heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + bool* planned = static_cast( + heap_caps_calloc(tensor_count, sizeof(bool), MALLOC_CAP_8BIT)); + struct PlanBuffers { + void* buffers[4]; + ~PlanBuffers() { + for (void* buffer : buffers) { + if (buffer != nullptr) + heap_caps_free(buffer); + } + } + } plan_buffers{{first, last, bytes, planned}}; + if (first == nullptr || last == nullptr || bytes == nullptr || + planned == nullptr) { + std::printf("ERROR: unable to allocate memory-planner tables\n"); + return false; + } + for (uint32_t i = 0; i < tensor_count; ++i) { + first[i] = + (tensors_[i].flags & TENSOR_FLAG_GRAPH_INPUT) != 0 ? -1 : INT32_MAX; + last[i] = + (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; + bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); + tensor_offsets_[i] = SIZE_MAX; + } + for (int32_t op = 0; op < op_count; ++op) { + const OperationRecord& record = operations_[op]; + for (uint8_t j = 0; j < record.num_inputs; ++j) { + const uint32_t id = record.inputs[j]; + if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) == 0 && last[id] < op) + last[id] = op; + } + for (uint8_t j = 0; j < record.num_outputs; ++j) { + const uint32_t id = record.outputs[j]; + if (first[id] > op) + first[id] = op; + if (last[id] < op) + last[id] = op; + } + } + + size_t arena_size = 0; + for (uint32_t step = 0; step < tensor_count; ++step) { + int32_t best = -1; + for (uint32_t i = 0; i < tensor_count; ++i) { + if (planned[i] || (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0) + continue; + if (first[i] == INT32_MAX) + continue; + if (best < 0 || bytes[i] > bytes[best]) + best = static_cast(i); + } + if (best < 0) + break; + planned[best] = true; + size_t candidate = 0; + bool moved = true; + while (moved) { + moved = false; + for (uint32_t j = 0; j < tensor_count; ++j) { + if (j == static_cast(best) || tensor_offsets_[j] == SIZE_MAX) + continue; + const bool lifetime_overlap = + first[j] <= last[best] && first[best] <= last[j]; + if (!lifetime_overlap) + continue; + const size_t other_begin = tensor_offsets_[j]; + const size_t other_end = other_begin + bytes[j]; + if (candidate < other_end && other_begin < candidate + bytes[best]) { + candidate = other_end; + moved = true; + } + } + } + tensor_offsets_[best] = candidate; + if (candidate + bytes[best] > arena_size) + arena_size = candidate + bytes[best]; + } + for (uint32_t i = 0; i < tensor_count; ++i) { + if ((tensors_[i].flags & TENSOR_FLAG_CONSTANT) == 0 && + tensor_offsets_[i] == SIZE_MAX) { + std::printf( + "ERROR: tensor %u is never produced or consumed\n", + static_cast(i)); + return false; + } + } + + arena_size_ = arena_size; + arena_ = static_cast(allocate_preferring_internal( + arena_size_, + options_.prefer_internal_arena, + options_.internal_reserve_bytes, + &arena_internal_)); + if (arena_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte activation arena\n", + static_cast(arena_size_)); + return false; + } + std::memset(arena_, 0, arena_size_); + return true; +} + +bool Int8Interpreter::prepare_operations() { + const uint32_t op_count = graph_.header.num_operations; + size_t param_words = 0; + for (uint32_t i = 0; i < op_count; ++i) { + const OperationRecord& op = operations_[i]; + const OpCode code = static_cast(op.opcode); + if (code == OpCode::CONV2D || code == OpCode::DEPTHWISE_CONV2D || + code == OpCode::LINEAR || code == OpCode::TRANSPOSE_CONV2D) { + param_words += 2U * tensor_element_count(tensors_[op.inputs[2]]); + } else if (code == OpCode::SOFTMAX) { + param_words += 256U; + } + } + params_size_ = param_words * sizeof(int32_t); + if (params_size_ != 0) { + params_ = static_cast(allocate_preferring_internal( + params_size_, true, options_.internal_reserve_bytes, nullptr)); + if (params_ == nullptr) { + std::printf("ERROR: unable to allocate requantization tables\n"); + return false; + } + } + + int32_t* cursor = params_; + size_t scratch_size = 0; + for (uint32_t i = 0; i < op_count; ++i) { + const OperationRecord& op = operations_[i]; + OpState& state = op_states_[i]; + state = {}; + state.opcode = static_cast(op.opcode); + state.input0 = op.inputs[0]; + state.input1 = op.num_inputs > 1 ? op.inputs[1] : 0; + state.output = op.outputs[0]; + const TensorRecord& input = tensors_[op.inputs[0]]; + const TensorRecord& output = tensors_[op.outputs[0]]; + state.input_zero_point = input.quant_zero_point; + state.output_zero_point = output.quant_zero_point; + state.elements = static_cast(tensor_element_count(output)); + const uint8_t* p = graph_.parameter_data + op.parameter_offset; + + switch (state.opcode) { + case OpCode::CONV2D: + case OpCode::DEPTHWISE_CONV2D: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + const uint32_t channels = static_cast(output.shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph_, op, channels, params)) + return false; + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + state.multipliers = cursor; + state.shifts = cursor + channels; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(params.multipliers + c * 4); + state.shifts[c] = read_le_i32(params.shifts + c * 4); + } + cursor += 2U * channels; + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + state.dilation_h = params.dilation_h; + state.dilation_w = params.dilation_w; + state.generic = params.dilation_h != 1 || params.dilation_w != 1; + state.pool[0] = params.kernel_h; + state.pool[1] = params.kernel_w; + state.pool[2] = params.stride_h; + state.pool[3] = params.stride_w; + state.pool[4] = params.pad_h; + state.pool[5] = params.pad_w; + state.value0 = params.activation_min; + state.value1 = params.activation_max; + state.rows = params.groups; + if (state.generic) { + if (params.stride_h != 1 || params.stride_w != 1) + break; + state.transpose = true; + const int32_t phase_h = + (output.shape[2] + params.dilation_h - 1) / params.dilation_h; + const int32_t phase_w = + (output.shape[3] + params.dilation_w - 1) / params.dilation_w; + data_dims_t tile_in{ + phase_w + params.kernel_w - 1, + phase_h + params.kernel_h - 1, + input.shape[1], + 1}; + data_dims_t tile_out{phase_w, phase_h, output.shape[1], 1}; + const size_t tile_bytes = static_cast(tile_in.width) * + tile_in.height * tile_in.channels + + static_cast(phase_w) * phase_h * output.shape[1]; + if (tile_bytes > phase_bytes_) + phase_bytes_ = tile_bytes; + if (state.opcode == OpCode::CONV2D) { + state.filter_dims = { + params.kernel_w, params.kernel_h, weight.shape[1], 1}; + state.conv.in_offset = -input.quant_zero_point; + state.conv.out_offset = output.quant_zero_point; + state.conv.stride = {1, 1}; + state.conv.padding = {0, 0}; + state.conv.dilation = {1, 1}; + state.conv.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_conv_scratch_size( + &tile_in, &state.filter_dims, &tile_out, &state.conv); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } else { + state.filter_dims = { + params.kernel_w, params.kernel_h, output.shape[1], 1}; + state.dw.in_offset = -input.quant_zero_point; + state.dw.out_offset = output.quant_zero_point; + state.dw.ch_mult = output.shape[1] / input.shape[1]; + state.dw.stride = {1, 1}; + state.dw.padding = {0, 0}; + state.dw.dilation = {1, 1}; + state.dw.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_depthwise_conv_scratch_size( + &tile_in, &state.filter_dims, &tile_out, &state.dw); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } + break; + } + if (state.opcode == OpCode::CONV2D) { + state.filter_dims = { + params.kernel_w, params.kernel_h, weight.shape[1], 1}; + state.conv.in_offset = -input.quant_zero_point; + state.conv.out_offset = output.quant_zero_point; + state.conv.stride = {params.stride_w, params.stride_h}; + state.conv.padding = {params.pad_w, params.pad_h}; + state.conv.dilation = {1, 1}; + state.conv.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_conv_scratch_size( + &state.input_dims, + &state.filter_dims, + &state.output_dims, + &state.conv); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } else { + state.filter_dims = { + params.kernel_w, params.kernel_h, output.shape[1], 1}; + state.dw.in_offset = -input.quant_zero_point; + state.dw.out_offset = output.quant_zero_point; + state.dw.ch_mult = output.shape[1] / input.shape[1]; + state.dw.stride = {params.stride_w, params.stride_h}; + state.dw.padding = {params.pad_w, params.pad_h}; + state.dw.dilation = {1, 1}; + state.dw.activation = {params.activation_min, params.activation_max}; + const int needed = esp_nn_get_depthwise_conv_scratch_size( + &state.input_dims, + &state.filter_dims, + &state.output_dims, + &state.dw); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } + break; + } + case OpCode::RELU6: + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + break; + case OpCode::ADD: + if (!read_int8_add_params(graph_, op, state.add)) + return false; + state.value0 = tensors_[op.inputs[1]].quant_zero_point; + break; + case OpCode::COPY: + state.input_count = op.num_inputs; + for (uint8_t k = 0; k < op.num_inputs; ++k) + state.inputs[k] = op.inputs[k]; + state.table = p; + break; + case OpCode::UPSAMPLE: + state.table = p; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + state.value0 = read_le_i32(p); + break; + case OpCode::TRANSPOSE_CONV2D: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + const uint32_t channels = static_cast(output.shape[1]); + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + for (int k = 0; k < 6; ++k) + state.pool[k] = read_le_i32(p + 4 * k); + state.dilation_h = read_le_i32(p + 32); + state.dilation_w = read_le_i32(p + 36); + state.value0 = read_le_i32(p + 44); + state.value1 = read_le_i32(p + 48); + state.multipliers = cursor; + state.shifts = cursor + channels; + const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + const uint8_t* shifts = multipliers + channels * 4U; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(multipliers + c * 4); + state.shifts[c] = read_le_i32(shifts + c * 4); + } + cursor += 2U * channels; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + break; + } + case OpCode::AVGPOOL2D: { + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + const int32_t needed = esp_nn_get_mean_scratch_size( + input.shape[2], input.shape[3], input.shape[1]); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + break; + } + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: + for (int k = 0; k < 6; ++k) + state.pool[k] = p[k]; + state.value0 = read_le_i32(p + 8); + state.value1 = read_le_i32(p + 12); + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + break; + case OpCode::LUT: + state.weights = reinterpret_cast(p); + break; + case OpCode::MUL: + state.value0 = tensors_[op.inputs[1]].quant_zero_point; + state.add.output_mult = read_le_i32(p); + state.add.output_shift = read_le_i32(p + 4); + state.add.activation_min = read_le_i32(p + 8); + state.add.activation_max = read_le_i32(p + 12); + state.transpose = read_le_i32(p + 16) == MUL_MODE_CHANNEL_BROADCAST; + state.cols = input.rank == 4 ? input.shape[1] + : static_cast(state.elements); + state.rows = static_cast(state.elements) / state.cols; + break; + case OpCode::SOFTMAX: + state.multipliers = cursor; + for (int k = 0; k < 256; ++k) + state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); + cursor += 256; + state.cols = input.shape[1]; + state.rows = static_cast(state.elements) / state.cols; + break; + case OpCode::CONCAT: + state.input_count = op.num_inputs; + for (uint8_t k = 0; k < op.num_inputs; ++k) + state.inputs[k] = op.inputs[k]; + state.rows = output.rank == 4 ? output.shape[2] * output.shape[3] : 1; + state.cols = output.shape[1]; + break; + case OpCode::FLATTEN: + state.transpose = + input.rank == 4 && input.shape[2] * input.shape[3] > 1; + state.input_dims = input.rank == 4 + ? data_dims_t{input.shape[3], input.shape[2], input.shape[1], 1} + : data_dims_t{ + 1, 1, static_cast(tensor_element_count(input)), 1}; + break; + case OpCode::LINEAR: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + state.rows = weight.shape[0]; + state.cols = weight.shape[1]; + state.batches = static_cast( + tensor_element_count(input) / static_cast(weight.shape[1])); + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + state.multipliers = cursor; + state.shifts = cursor + state.rows; + const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; + const uint8_t* shifts = + multipliers + static_cast(state.rows) * 4U; + for (int32_t r = 0; r < state.rows; ++r) { + state.multipliers[r] = read_le_i32(multipliers + r * 4); + state.shifts[r] = read_le_i32(shifts + r * 4); + } + cursor += 2 * state.rows; + break; + } + default: + return false; + } + } + + if (phase_bytes_ != 0) { + phase_buffer_ = static_cast(allocate_preferring_internal( + phase_bytes_, true, options_.internal_reserve_bytes, nullptr)); + if (phase_buffer_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte dilated-convolution tile buffer\n", + static_cast(phase_bytes_)); + return false; + } + } + scratch_size_ = scratch_size; + if (scratch_size_ != 0) { + scratch_ = allocate_preferring_internal( + scratch_size_, true, options_.internal_reserve_bytes, nullptr); + if (scratch_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte ESP-NN scratch buffer\n", + static_cast(scratch_size_)); + return false; + } + } + return true; +} + +void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { + const uint8_t* old_end = old_base + size; + auto rebase = [&](const void* pointer) -> const uint8_t* { + const uint8_t* bytes = static_cast(pointer); + if (bytes >= old_base && bytes < old_end) + return graph_.data + (bytes - old_base); + return bytes; + }; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + OpState& state = op_states_[i]; + if (state.weights != nullptr) + state.weights = reinterpret_cast(rebase(state.weights)); + if (state.bias != nullptr) + state.bias = reinterpret_cast(rebase(state.bias)); + if (state.table != nullptr) + state.table = rebase(state.table); + } +} + +int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { + return reinterpret_cast(arena_ + tensor_offsets_[tensor_id]); +} + +const int8_t* Int8Interpreter::operand(uint32_t tensor_id) const { + if ((tensors_[tensor_id].flags & TENSOR_FLAG_CONSTANT) != 0) { + return reinterpret_cast( + tensor_constant_data(graph_, tensors_[tensor_id])); + } + return activation(tensor_id); +} + +const int8_t* Int8Interpreter::tensor_data(uint32_t tensor_id) const { + if (!loaded_ || tensor_id >= graph_.header.num_tensors) + return nullptr; + if ((tensors_[tensor_id].flags & TENSOR_FLAG_CONSTANT) != 0) { + return reinterpret_cast( + tensor_constant_data(graph_, tensors_[tensor_id])); + } + return activation(tensor_id); +} + +bool Int8Interpreter::tensor_info(uint32_t tensor_id, Int8TensorInfo& info) + const { + if (!loaded_ || tensor_id >= graph_.header.num_tensors) + return false; + const TensorRecord& tensor = tensors_[tensor_id]; + info = {}; + info.tensor_id = tensor_id; + info.rank = tensor.rank; + std::memcpy(info.shape, tensor.shape, sizeof(info.shape)); + info.bytes = tensor_data_size(tensor); + info.scale = tensor.quant_scale; + info.zero_point = tensor.quant_zero_point; + if (tensor.rank == 4) { + info.height = tensor.shape[2]; + info.width = tensor.shape[3]; + info.channels = tensor.shape[1]; + } else { + info.height = 1; + info.width = 1; + info.channels = static_cast(tensor_element_count(tensor)); + } + return true; +} + +bool Int8Interpreter::input_info(size_t index, Int8TensorInfo& info) const { + return index < input_count_ && tensor_info(inputs_[index], info); +} + +bool Int8Interpreter::output_info(size_t index, Int8TensorInfo& info) const { + return index < output_count_ && tensor_info(outputs_[index], info); +} + +int8_t* Int8Interpreter::input_data(size_t index) { + return loaded_ && index < input_count_ ? activation(inputs_[index]) : nullptr; +} + +const int8_t* Int8Interpreter::output_data(size_t index) const { + return loaded_ && index < output_count_ ? activation(outputs_[index]) + : nullptr; +} + +bool Int8Interpreter::quantize_input_nchw( + size_t index, + const float* values, + size_t count) { + Int8TensorInfo info{}; + if (values == nullptr || !input_info(index, info) || count != info.bytes) + return false; + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + int8_t* destination = input_data(index); + const size_t plane = static_cast(info.height) * info.width; + for (int32_t c = 0; c < info.channels; ++c) { + for (size_t s = 0; s < plane; ++s) { + destination[s * info.channels + c] = + quantize_value(values[c * plane + s], inverse, info.zero_point); + } + } + return true; +} + +bool Int8Interpreter::set_input_rgb888( + size_t index, + const uint8_t* rgb, + int32_t width, + int32_t height, + const float mean[3], + const float stddev[3]) { + Int8TensorInfo info{}; + if (rgb == nullptr || !input_info(index, info) || info.channels != 3 || + info.width != width || info.height != height) + return false; + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + int8_t lut[3][256]; + for (int c = 0; c < 3; ++c) { + for (int v = 0; v < 256; ++v) { + const float normalized = + (static_cast(v) / 255.0f - mean[c]) / stddev[c]; + lut[c][v] = quantize_value(normalized, inverse, info.zero_point); + } + } + int8_t* destination = input_data(index); + const size_t pixels = static_cast(width) * height; + for (size_t i = 0; i < pixels; ++i) { + destination[i * 3 + 0] = lut[0][rgb[i * 3 + 0]]; + destination[i * 3 + 1] = lut[1][rgb[i * 3 + 1]]; + destination[i * 3 + 2] = lut[2][rgb[i * 3 + 2]]; + } + return true; +} + +bool Int8Interpreter::dequantize_output( + size_t index, + float* values, + size_t count) const { + Int8TensorInfo info{}; + if (values == nullptr || !output_info(index, info) || count != info.bytes) + return false; + const int8_t* source = output_data(index); + for (size_t i = 0; i < count; ++i) { + values[i] = static_cast(source[i] - info.zero_point) * info.scale; + } + return true; +} + +int32_t Int8Interpreter::output_argmax(size_t index) const { + Int8TensorInfo info{}; + if (!output_info(index, info)) + return -1; + const int8_t* source = output_data(index); + int32_t best = 0; + for (size_t i = 1; i < info.bytes; ++i) { + if (source[i] > source[best]) + best = static_cast(i); + } + return best; +} + +int64_t Int8Interpreter::op_time_us(uint32_t op) const { + return loaded_ && op < graph_.header.num_operations ? op_states_[op].last_us + : 0; +} + +bool Int8Interpreter::run_operation(uint32_t index) { + OpState& s = op_states_[index]; + const bool reference = options_.kernel_mode == KernelMode::REFERENCE; + const int8_t* input = activation(s.input0); + int8_t* output = activation(s.output); + switch (s.opcode) { + case OpCode::CONV2D: + if (s.generic) { + run_generic_conv(s, input, output); + return true; + } + if (reference) { + esp_nn_conv_s8_ansi( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.conv, + &s.quant); + } else { + esp_nn_conv_s8( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.conv, + &s.quant); + } + return true; + case OpCode::DEPTHWISE_CONV2D: + if (s.generic) { + run_generic_conv(s, input, output); + return true; + } + if (reference) { + esp_nn_depthwise_conv_s8_ansi( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.dw, + &s.quant); + } else { + esp_nn_depthwise_conv_s8( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.dw, + &s.quant); + } + return true; + case OpCode::RELU6: + for (uint32_t i = 0; i < s.elements; ++i) { + int32_t value = input[i]; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + output[i] = static_cast(value); + } + return true; + case OpCode::ADD: { + input = operand(s.input0); + const int8_t* rhs = operand(s.input1); + if (reference) { + esp_nn_add_elementwise_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + s.add.input1_mult, + s.add.input2_mult, + s.add.input1_shift, + s.add.input2_shift, + s.add.left_shift, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } else { + esp_nn_add_elementwise_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + s.add.input1_mult, + s.add.input2_mult, + s.add.input1_shift, + s.add.input2_shift, + s.add.left_shift, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } + return true; + } + case OpCode::AVGPOOL2D: + if (reference) { + esp_nn_mean_nhwc_s8_ansi( + input, + output, + s.input_dims.height, + s.input_dims.width, + s.input_dims.channels, + s.input_zero_point, + s.output_zero_point, + s.value0, + s.value1); + } else { + esp_nn_mean_nhwc_s8( + input, + output, + s.input_dims.height, + s.input_dims.width, + s.input_dims.channels, + s.input_zero_point, + s.output_zero_point, + s.value0, + s.value1); + } + return true; + case OpCode::FLATTEN: + if (s.transpose) { + const int32_t plane = s.input_dims.height * s.input_dims.width; + const int32_t channels = s.input_dims.channels; + for (int32_t c = 0; c < channels; ++c) { + for (int32_t p = 0; p < plane; ++p) { + output[c * plane + p] = input[p * channels + c]; + } + } + } else { + std::memcpy(output, input, s.elements); + } + return true; + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: { + const uint16_t in_w = static_cast(s.input_dims.width); + const uint16_t in_h = static_cast(s.input_dims.height); + const uint16_t out_w = static_cast(s.output_dims.width); + const uint16_t out_h = static_cast(s.output_dims.height); + const uint16_t channels = static_cast(s.input_dims.channels); + const uint16_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], + sw = s.pool[3]; + const uint16_t ph = s.pool[4], pw = s.pool[5]; + if (s.opcode == OpCode::MAX_POOL2D) { + if (reference) { + esp_nn_max_pool_s8_ansi( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } else { + esp_nn_max_pool_s8( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } + } else if (reference) { + esp_nn_avg_pool_s8_ansi( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } else { + esp_nn_avg_pool_s8( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } + return true; + } + case OpCode::LUT: { + const uint8_t* table = reinterpret_cast(s.weights) + 128; + const uint32_t words = s.elements / 4; + const uint32_t* source = reinterpret_cast(input); + uint32_t* destination = reinterpret_cast(output); + for (uint32_t w = 0; w < words; ++w) { + const uint32_t packed = source[w]; + destination[w] = + static_cast(table[static_cast(packed)]) | + (static_cast(table[static_cast(packed >> 8)]) + << 8) | + (static_cast(table[static_cast(packed >> 16)]) + << 16) | + (static_cast(table[static_cast(packed >> 24)]) + << 24); + } + for (uint32_t i = words * 4; i < s.elements; ++i) { + output[i] = static_cast(table[input[i]]); + } + return true; + } + case OpCode::MUL: { + input = operand(s.input0); + const int8_t* rhs = operand(s.input1); + if (s.transpose) { + if (reference) { + esp_nn_mul_broadcast_channel_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + s.rows, + s.cols); + } else { + esp_nn_mul_broadcast_channel_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + s.rows, + s.cols); + } + } else if (reference) { + esp_nn_mul_elementwise_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } else { + esp_nn_mul_elementwise_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } + return true; + } + case OpCode::SOFTMAX: { + const uint32_t* table = reinterpret_cast(s.multipliers); + for (int32_t r = 0; r < s.rows; ++r) { + const int8_t* row = input + r * s.cols; + int8_t* destination = output + r * s.cols; + int32_t maximum = row[0]; + for (int32_t c = 1; c < s.cols; ++c) { + if (row[c] > maximum) + maximum = row[c]; + } + uint64_t total = 0; + for (int32_t c = 0; c < s.cols; ++c) + total += table[maximum - row[c]]; + for (int32_t c = 0; c < s.cols; ++c) { + const uint64_t numerator = 512ULL * table[maximum - row[c]] + total; + const int32_t value = + static_cast(numerator / (2ULL * total)) + + SOFTMAX_OUTPUT_ZERO_POINT; + destination[c] = static_cast(value > 127 ? 127 : value); + } + } + return true; + } + case OpCode::CONCAT: { + int32_t offset = 0; + for (uint8_t k = 0; k < s.input_count; ++k) { + const int32_t channels = tensors_[s.inputs[k]].shape[1]; + const int8_t* source = activation(s.inputs[k]); + for (int32_t r = 0; r < s.rows; ++r) { + std::memcpy( + output + r * s.cols + offset, source + r * channels, channels); + } + offset += channels; + } + return true; + } + case OpCode::LINEAR: + for (int32_t b = 0; b < s.batches; ++b) { + const int8_t* row = input + static_cast(b) * s.cols; + int8_t* destination = output + static_cast(b) * s.rows; + if (reference) { + esp_nn_fully_connected_per_ch_s8_ansi( + row, + -s.input_zero_point, + static_cast(s.cols), + s.weights, + 0, + s.bias, + destination, + static_cast(s.rows), + s.output_zero_point, + s.shifts, + s.multipliers, + s.value0, + s.value1); + } else { + esp_nn_fully_connected_per_ch_s8( + row, + -s.input_zero_point, + static_cast(s.cols), + s.weights, + 0, + s.bias, + destination, + static_cast(s.rows), + s.output_zero_point, + s.shifts, + s.multipliers, + s.value0, + s.value1); + } + } + return true; + case OpCode::COPY: + run_copy(s, output); + return true; + case OpCode::UPSAMPLE: + run_upsample(s, input, output); + return true; + case OpCode::TRANSPOSE_CONV2D: + run_transposed_conv(s, input, output); + return true; + default: + return false; + } +} + +void Int8Interpreter::run_dilated_by_phases( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const bool reference = options_.kernel_mode == KernelMode::REFERENCE; + const int8_t pad_value = clamp_to_int8(s.input_zero_point); + for (int32_t qy = 0; qy < dh; ++qy) { + const int32_t rows = (out_h - qy + dh - 1) / dh; + if (rows <= 0) + continue; + const int32_t ty = qy - ph; + const int32_t ry = ((ty % dh) + dh) % dh; + const int32_t by = (ty - ry) / dh; + for (int32_t qx = 0; qx < dw; ++qx) { + const int32_t cols = (out_w - qx + dw - 1) / dw; + if (cols <= 0) + continue; + const int32_t tx = qx - pw; + const int32_t rx = ((tx % dw) + dw) % dw; + const int32_t bx = (tx - rx) / dw; + const int32_t tile_h = rows + kh - 1; + const int32_t tile_w = cols + kw - 1; + int8_t* tile = phase_buffer_; + int8_t* result = + phase_buffer_ + static_cast(tile_h) * tile_w * in_c; + for (int32_t i = 0; i < tile_h; ++i) { + const int32_t iy = ry + dh * (by + i); + int8_t* row = tile + static_cast(i) * tile_w * in_c; + for (int32_t j = 0; j < tile_w; ++j) { + const int32_t ix = rx + dw * (bx + j); + int8_t* pixel = row + static_cast(j) * in_c; + if (iy < 0 || iy >= in_h || ix < 0 || ix >= in_w) { + std::memset(pixel, pad_value, in_c); + } else { + std::memcpy( + pixel, + input + (static_cast(iy) * in_w + ix) * in_c, + in_c); + } + } + } + data_dims_t tile_dims{tile_w, tile_h, in_c, 1}; + data_dims_t result_dims{cols, rows, out_c, 1}; + if (s.opcode == OpCode::CONV2D) { + if (reference) { + esp_nn_conv_s8_ansi( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.conv, + &s.quant); + } else { + esp_nn_conv_s8( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.conv, + &s.quant); + } + } else if (reference) { + esp_nn_depthwise_conv_s8_ansi( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.dw, + &s.quant); + } else { + esp_nn_depthwise_conv_s8( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.dw, + &s.quant); + } + for (int32_t i = 0; i < rows; ++i) { + const int32_t oy = qy + dh * i; + for (int32_t j = 0; j < cols; ++j) { + const int32_t ox = qx + dw * j; + std::memcpy( + output + (static_cast(oy) * out_w + ox) * out_c, + result + (static_cast(i) * cols + j) * out_c, + out_c); + } + } + } + } +} + +void Int8Interpreter::run_generic_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const { + if (s.transpose) { + run_dilated_by_phases(s, input, output); + return; + } + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], sw = s.pool[3]; + const int32_t ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const bool depthwise = s.opcode == OpCode::DEPTHWISE_CONV2D; + const int32_t groups = depthwise ? in_c : s.rows; + const int32_t group_in = depthwise ? 1 : in_c / groups; + const int32_t group_out = out_c / groups; + const int32_t in_offset = -s.input_zero_point; + for (int32_t oy = 0; oy < out_h; ++oy) { + for (int32_t ox = 0; ox < out_w; ++ox) { + int8_t* destination = + output + (static_cast(oy) * out_w + ox) * out_c; + for (int32_t oc = 0; oc < out_c; ++oc) { + const int32_t g = oc / group_out; + int32_t acc = s.bias != nullptr ? s.bias[oc] : 0; + for (int32_t ky = 0; ky < kh; ++ky) { + const int32_t iy = oy * sh - ph + ky * dh; + if (iy < 0 || iy >= in_h) + continue; + for (int32_t kx = 0; kx < kw; ++kx) { + const int32_t ix = ox * sw - pw + kx * dw; + if (ix < 0 || ix >= in_w) + continue; + const int8_t* pixel = + input + (static_cast(iy) * in_w + ix) * in_c; + if (depthwise) { + const int32_t ic = oc / (out_c / in_c); + acc += (pixel[ic] + in_offset) * + s.weights[(static_cast(ky) * kw + kx) * out_c + oc]; + } else { + const int8_t* filter = s.weights + + ((static_cast(oc) * kh + ky) * kw + kx) * group_in; + const int8_t* source = pixel + g * group_in; + for (int32_t ic = 0; ic < group_in; ++ic) { + acc += (source[ic] + in_offset) * filter[ic]; + } + } + } + } + int32_t value = requantize(acc, s.multipliers[oc], s.shifts[oc]) + + s.output_zero_point; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + destination[oc] = static_cast(value); + } + } + } +} + +void Int8Interpreter::run_copy(const OpState& s, int8_t* output) const { + const uint8_t* p = s.table; + const int32_t segments = read_le_i32(p); + const int32_t flags = read_le_i32(p + 4); + if ((flags & COPY_FLAG_FILL) != 0) { + std::memset( + output, read_le_i32(p + 8), tensor_data_size(tensors_[s.output])); + } + for (int32_t k = 0; k < segments; ++k) { + const uint8_t* q = + p + COPY_HEADER_SIZE + static_cast(k) * COPY_SEGMENT_SIZE; + const int8_t* source = operand(s.inputs[read_le_i32(q)]); + const int32_t rank = read_le_i32(q + 4); + const int32_t in_base = read_le_i32(q + 8); + const int32_t out_base = read_le_i32(q + 12); + int32_t dims[COPY_MAX_RANK]; + int32_t in_strides[COPY_MAX_RANK]; + int32_t out_strides[COPY_MAX_RANK]; + for (int32_t d = 0; d < static_cast(COPY_MAX_RANK); ++d) { + const int32_t from = d - (static_cast(COPY_MAX_RANK) - rank); + dims[d] = from >= 0 ? read_le_i32(q + 16 + 4 * from) : 1; + in_strides[d] = from >= 0 ? read_le_i32(q + 40 + 4 * from) : 0; + out_strides[d] = from >= 0 ? read_le_i32(q + 64 + 4 * from) : 0; + } + const bool requant = read_le_i32(q + 88) != 0; + const int32_t multiplier = read_le_i32(q + 92); + const int32_t shift = read_le_i32(q + 96); + const int32_t in_zp = read_le_i32(q + 100); + const int32_t out_zp = read_le_i32(q + 104); + const int32_t inner = dims[5]; + const bool contiguous = + in_strides[5] == 1 && out_strides[5] == 1 && !requant; + for (int32_t a = 0; a < dims[0]; ++a) + for (int32_t b = 0; b < dims[1]; ++b) + for (int32_t c = 0; c < dims[2]; ++c) + for (int32_t d = 0; d < dims[3]; ++d) + for (int32_t e = 0; e < dims[4]; ++e) { + const int32_t in_index = in_base + a * in_strides[0] + + b * in_strides[1] + c * in_strides[2] + d * in_strides[3] + + e * in_strides[4]; + const int32_t out_index = out_base + a * out_strides[0] + + b * out_strides[1] + c * out_strides[2] + d * out_strides[3] + + e * out_strides[4]; + if (contiguous) { + std::memcpy(output + out_index, source + in_index, inner); + continue; + } + const int8_t* from = source + in_index; + int8_t* to = output + out_index; + for (int32_t f = 0; f < inner; ++f) { + int32_t value = from[f * in_strides[5]]; + if (requant) + value = clamp_to_int8( + requantize(value - in_zp, multiplier, shift) + out_zp); + to[f * out_strides[5]] = static_cast(value); + } + } + } +} + +void Int8Interpreter::run_upsample( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_w = s.input_dims.width; + const int32_t channels = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const uint8_t* y_table = s.table + UPSAMPLE_HEADER_SIZE; + const uint8_t* x_table = y_table + 12U * static_cast(out_h); + const int32_t one = 1 << UPSAMPLE_WEIGHT_BITS; + const int32_t half = 1 << (2 * UPSAMPLE_WEIGHT_BITS - 1); + const bool nearest = s.value0 == UPSAMPLE_MODE_NEAREST; + for (int32_t oy = 0; oy < out_h; ++oy) { + const int32_t y0 = read_le_i32(y_table + 12 * oy); + const int32_t y1 = read_le_i32(y_table + 12 * oy + 4); + const int32_t wy = read_le_i32(y_table + 12 * oy + 8); + const int8_t* row0 = input + static_cast(y0) * in_w * channels; + const int8_t* row1 = input + static_cast(y1) * in_w * channels; + int8_t* destination = output + static_cast(oy) * out_w * channels; + for (int32_t ox = 0; ox < out_w; ++ox) { + const int32_t x0 = read_le_i32(x_table + 12 * ox); + const int32_t x1 = read_le_i32(x_table + 12 * ox + 4); + const int32_t wx = read_le_i32(x_table + 12 * ox + 8); + int8_t* pixel = destination + static_cast(ox) * channels; + if (nearest) { + std::memcpy(pixel, row0 + static_cast(x0) * channels, channels); + continue; + } + const int8_t* a = row0 + static_cast(x0) * channels; + const int8_t* b = row0 + static_cast(x1) * channels; + const int8_t* c = row1 + static_cast(x0) * channels; + const int8_t* d = row1 + static_cast(x1) * channels; + for (int32_t ch = 0; ch < channels; ++ch) { + const int32_t top = a[ch] * (one - wx) + b[ch] * wx; + const int32_t bottom = c[ch] * (one - wx) + d[ch] * wx; + const int32_t total = top * (one - wy) + bottom * wy + half; + pixel[ch] = clamp_to_int8(total >> (2 * UPSAMPLE_WEIGHT_BITS)); + } + } + } +} + +void Int8Interpreter::run_transposed_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], sw = s.pool[3]; + const int32_t ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const int32_t in_offset = -s.input_zero_point; + for (int32_t oy = 0; oy < out_h; ++oy) { + for (int32_t ox = 0; ox < out_w; ++ox) { + int8_t* destination = + output + (static_cast(oy) * out_w + ox) * out_c; + for (int32_t oc = 0; oc < out_c; ++oc) { + int32_t acc = s.bias[oc]; + for (int32_t ky = 0; ky < kh; ++ky) { + const int32_t ty = oy + ph - ky * dh; + if (ty < 0 || ty % sh != 0) + continue; + const int32_t iy = ty / sh; + if (iy >= in_h) + continue; + for (int32_t kx = 0; kx < kw; ++kx) { + const int32_t tx = ox + pw - kx * dw; + if (tx < 0 || tx % sw != 0) + continue; + const int32_t ix = tx / sw; + if (ix >= in_w) + continue; + const int8_t* pixel = + input + (static_cast(iy) * in_w + ix) * in_c; + const int8_t* filter = s.weights + + ((static_cast(oc) * kh + ky) * kw + kx) * in_c; + for (int32_t ic = 0; ic < in_c; ++ic) + acc += (pixel[ic] + in_offset) * filter[ic]; + } + } + int32_t value = requantize(acc, s.multipliers[oc], s.shifts[oc]) + + s.output_zero_point; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + destination[oc] = static_cast(value); + } + } + } +} + +bool Int8Interpreter::invoke() { + if (!loaded_) + return false; + if (scratch_ != nullptr) { + esp_nn_set_conv_scratch_buf(scratch_); + esp_nn_set_depthwise_conv_scratch_buf(scratch_); + esp_nn_set_mean_scratch_buf(scratch_); + } + int64_t compute_us = 0; + int64_t last_yield = esp_timer_get_time(); + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + const int64_t started = esp_timer_get_time(); + if (!run_operation(i)) { + std::printf( + "ERROR: operation %u (%s) failed\n", + static_cast(i), + opcode_name(op_states_[i].opcode)); + return false; + } + const int64_t finished = esp_timer_get_time(); + OpState& state = op_states_[i]; + state.last_us = finished - started; + state.total_us += state.last_us; + ++state.calls; + compute_us += state.last_us; + if (finished - last_yield > YIELD_INTERVAL_US) { + vTaskDelay(1); + last_yield = esp_timer_get_time(); + } + } + last_invoke_us_ = compute_us; + return true; +} + +void Int8Interpreter::print_summary() const { + if (!loaded_) + return; + std::printf( + "Int8 model: %u tensors, %u operations, %u constant bytes\n", + static_cast(graph_.header.num_tensors), + static_cast(graph_.header.num_operations), + static_cast(graph_.header.constant_size)); + std::printf( + "Activation arena: %u bytes in %s\n", + static_cast(arena_size_), + arena_internal_ ? "internal RAM" : "PSRAM"); + std::printf( + "ESP-NN scratch: %u bytes; requant tables: %u bytes; constants %s\n", + static_cast(scratch_size_), + static_cast(params_size_), + blob_copy_ != nullptr ? "copied to RAM" : "read in place from flash"); + for (size_t i = 0; i < input_count_; ++i) { + Int8TensorInfo info{}; + input_info(i, info); + std::printf( + "Input %u: tensor %u HWC=%ldx%ldx%ld scale=%.9g zero_point=%ld\n", + static_cast(i), + static_cast(info.tensor_id), + static_cast(info.height), + static_cast(info.width), + static_cast(info.channels), + info.scale, + static_cast(info.zero_point)); + } + for (size_t i = 0; i < output_count_; ++i) { + Int8TensorInfo info{}; + output_info(i, info); + std::printf( + "Output %u: tensor %u elements=%u scale=%.9g zero_point=%ld\n", + static_cast(i), + static_cast(info.tensor_id), + static_cast(info.bytes), + info.scale, + static_cast(info.zero_point)); + } +} + +void Int8Interpreter::print_profile() const { + if (!loaded_) + return; + int64_t per_type[OPCODE_COUNT]{}; + uint32_t count_per_type[OPCODE_COUNT]{}; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + const OpState& s = op_states_[i]; + const unsigned type = static_cast(s.opcode); + if (type < OPCODE_COUNT) { + per_type[type] += s.last_us; + ++count_per_type[type]; + } + std::printf( + " op %2u %-16s out=%ldx%ldx%ld %8lld us\n", + static_cast(i), + opcode_name(s.opcode), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[2] : 1), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[3] : 1), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[1] + : static_cast(s.elements)), + static_cast(s.last_us)); + } + for (unsigned t = 0; t < OPCODE_COUNT; ++t) { + if (count_per_type[t] == 0) + continue; + std::printf( + " %-16s x%-3u %8lld us\n", + opcode_name(static_cast(t)), + static_cast(count_per_type[t]), + static_cast(per_type[t])); + } +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp new file mode 100644 index 00000000000..03674f98a81 --- /dev/null +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -0,0 +1,1050 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +uint16_t read_u16(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8); +} + +uint32_t read_u32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +int32_t read_i32(const uint8_t* p) { + return static_cast(read_u32(p)); +} + +bool range_valid(size_t offset, size_t length, size_t total) { + return offset <= total && length <= total - offset; +} + +bool valid_dtype(uint8_t dtype) { + return dtype >= static_cast(DataType::FLOAT32) && + dtype <= static_cast(DataType::INT32); +} + +bool valid_opcode(uint8_t opcode, uint32_t version) { + if (version == GRAPH_ESP_NN_INT8_IO_VERSION) { + return opcode <= static_cast(OpCode::TRANSPOSE_CONV2D); + } + return opcode <= static_cast(OpCode::SOFTMAX); +} + +bool valid_tensor_reference(const GraphView& graph, uint32_t id) { + return id < graph.header.num_tensors; +} + +bool same_shape(const TensorRecord& a, const TensorRecord& b) { + return a.rank == b.rank && + std::memcmp(a.shape, b.shape, sizeof(a.shape)) == 0; +} + +bool same_quantization(const TensorRecord& a, const TensorRecord& b) { + return a.quant_scale == b.quant_scale && + a.quant_zero_point == b.quant_zero_point; +} + +bool valid_requantization(int32_t multiplier, int32_t shift) { + return multiplier > 0 && shift >= -31 && shift <= 30; +} + +bool valid_activation_range(int32_t low, int32_t high) { + return low >= -128 && high <= 127 && low <= high; +} + +bool is_int8_activation(const TensorRecord& tensor) { + return tensor.dtype == static_cast(DataType::INT8) && + (tensor.flags & TENSOR_FLAG_CONSTANT) == 0 && + (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; +} + +bool is_int8_value(const TensorRecord& tensor) { + return tensor.dtype == static_cast(DataType::INT8) && + (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; +} + +bool valid_layout(const TensorRecord& tensor) { + return tensor.rank >= 1 && tensor.rank <= 4 && + (tensor.rank != 4 || tensor.shape[0] == 1); +} + +bool is_constant_of(const TensorRecord& tensor, DataType dtype) { + return tensor.dtype == static_cast(dtype) && + (tensor.flags & TENSOR_FLAG_CONSTANT) != 0 && + (tensor.flags & TENSOR_FLAG_QUANTIZED) == 0; +} + +bool validate_int8_conv( + const GraphView& graph, + const OperationRecord& op, + bool depthwise) { + if (op.num_inputs != 3 || op.num_outputs != 1) + return false; + TensorRecord input{}; + TensorRecord weight{}; + TensorRecord bias{}; + TensorRecord output{}; + if (!read_tensor(graph, op.inputs[0], input) || + !read_tensor(graph, op.inputs[1], weight) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], output)) + return false; + if (!is_int8_activation(input) || !is_int8_activation(output) || + !is_constant_of(weight, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || input.rank != 4 || + weight.rank != 4 || bias.rank != 1 || output.rank != 4 || + input.shape[0] != 1 || output.shape[0] != 1 || + weight.shape[0] != output.shape[1] || bias.shape[0] != output.shape[1]) + return false; + + const uint32_t channels = static_cast(output.shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph, op, channels, params)) + return false; + if (params.dilation_h <= 0 || params.dilation_w <= 0) + return false; + if (graph.header.version == GRAPH_ESP_NN_INT8_VERSION && + (params.dilation_h != 1 || params.dilation_w != 1)) + return false; + const int32_t effective_h = params.dilation_h * (params.kernel_h - 1) + 1; + const int32_t effective_w = params.dilation_w * (params.kernel_w - 1) + 1; + + if (depthwise) { + if (params.groups != input.shape[1] || weight.shape[1] != 1 || + output.shape[1] % input.shape[1] != 0) + return false; + } else if ( + params.groups <= 0 || + static_cast(weight.shape[1]) * params.groups != input.shape[1] || + output.shape[1] % params.groups != 0) { + return false; + } + if (params.kernel_h != weight.shape[2] || + params.kernel_w != weight.shape[3] || params.stride_h <= 0 || + params.stride_w <= 0 || params.pad_h >= effective_h || + params.pad_w >= effective_w || + !valid_activation_range(params.activation_min, params.activation_max)) + return false; + + const int64_t expected_h = + (static_cast(input.shape[2]) + 2 * params.pad_h - effective_h) / + params.stride_h + + 1; + const int64_t expected_w = + (static_cast(input.shape[3]) + 2 * params.pad_w - effective_w) / + params.stride_w + + 1; + if (output.shape[2] != expected_h || output.shape[3] != expected_w) + return false; + if (input.shape[1] > 65535 || input.shape[2] > 65535 || + input.shape[3] > 65535 || output.shape[1] > 65535) + return false; + for (uint32_t i = 0; i < channels; ++i) { + if (!valid_requantization( + read_i32(params.multipliers + i * 4), + read_i32(params.shifts + i * 4))) + return false; + } + return true; +} + +bool copy_extent_valid( + int32_t rank, + const int32_t* dims, + const int32_t* strides, + int32_t offset, + size_t limit) { + int64_t low = offset; + int64_t high = offset; + for (int32_t d = 0; d < rank; ++d) { + if (dims[d] <= 0) + return false; + const int64_t span = static_cast(dims[d] - 1) * strides[d]; + if (span < 0) + low += span; + else + high += span; + } + return low >= 0 && high < static_cast(limit); +} + +bool validate_copy(const GraphView& graph, const OperationRecord& op) { + TensorRecord out{}; + if (op.num_outputs != 1 || op.parameter_size < COPY_HEADER_SIZE || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(out) || + !valid_layout(out)) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + const int32_t segments = read_i32(p); + const int32_t flags = read_i32(p + 4); + const int32_t fill = read_i32(p + 8); + if (segments <= 0 || (flags & ~COPY_FLAG_FILL) != 0 || fill < -128 || + fill > 127 || + op.parameter_size != + COPY_HEADER_SIZE + + static_cast(segments) * COPY_SEGMENT_SIZE) { + return false; + } + TensorRecord inputs[6]{}; + for (uint8_t j = 0; j < op.num_inputs; ++j) { + if (!read_tensor(graph, op.inputs[j], inputs[j]) || + !is_int8_value(inputs[j]) || !valid_layout(inputs[j])) + return false; + } + const size_t out_count = tensor_element_count(out); + for (int32_t k = 0; k < segments; ++k) { + const uint8_t* q = + p + COPY_HEADER_SIZE + static_cast(k) * COPY_SEGMENT_SIZE; + const int32_t index = read_i32(q); + const int32_t rank = read_i32(q + 4); + const int32_t in_offset = read_i32(q + 8); + const int32_t out_offset = read_i32(q + 12); + int32_t dims[COPY_MAX_RANK]; + int32_t in_strides[COPY_MAX_RANK]; + int32_t out_strides[COPY_MAX_RANK]; + for (size_t d = 0; d < COPY_MAX_RANK; ++d) { + dims[d] = read_i32(q + 16 + 4 * d); + in_strides[d] = read_i32(q + 40 + 4 * d); + out_strides[d] = read_i32(q + 64 + 4 * d); + } + const int32_t requant = read_i32(q + 88); + if (index < 0 || index >= op.num_inputs || rank <= 0 || + rank > static_cast(COPY_MAX_RANK) || + (requant != 0 && requant != 1)) + return false; + const TensorRecord& source = inputs[index]; + if (!copy_extent_valid( + rank, dims, in_strides, in_offset, tensor_element_count(source)) || + !copy_extent_valid(rank, dims, out_strides, out_offset, out_count)) + return false; + if (requant != 0) { + if (!valid_requantization(read_i32(q + 92), read_i32(q + 96)) || + read_i32(q + 100) != source.quant_zero_point || + read_i32(q + 104) != out.quant_zero_point) + return false; + } else if (!same_quantization(source, out)) { + return false; + } + } + return true; +} + +bool validate_upsample(const GraphView& graph, const OperationRecord& op) { + TensorRecord a{}; + TensorRecord out{}; + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size < UPSAMPLE_HEADER_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_quantization(a, out) || a.rank != 4 || + out.rank != 4 || a.shape[0] != 1 || out.shape[0] != 1 || + a.shape[1] != out.shape[1]) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + const int32_t mode = read_i32(p); + const int32_t in_h = read_i32(p + 4); + const int32_t in_w = read_i32(p + 8); + const int32_t out_h = read_i32(p + 12); + const int32_t out_w = read_i32(p + 16); + if ((mode != UPSAMPLE_MODE_NEAREST && mode != UPSAMPLE_MODE_BILINEAR) || + in_h != a.shape[2] || in_w != a.shape[3] || out_h != out.shape[2] || + out_w != out.shape[3] || + op.parameter_size != + UPSAMPLE_HEADER_SIZE + 12U * static_cast(out_h + out_w)) { + return false; + } + const int32_t one = 1 << UPSAMPLE_WEIGHT_BITS; + const uint8_t* table = p + UPSAMPLE_HEADER_SIZE; + for (int32_t i = 0; i < out_h + out_w; ++i) { + const int32_t limit = i < out_h ? in_h : in_w; + const int32_t i0 = read_i32(table + 12 * i); + const int32_t i1 = read_i32(table + 12 * i + 4); + const int32_t weight = read_i32(table + 12 * i + 8); + if (i0 < 0 || i0 >= limit || i1 < 0 || i1 >= limit || weight < 0 || + weight > one) + return false; + } + return true; +} + +bool validate_transposed_conv( + const GraphView& graph, + const OperationRecord& op) { + TensorRecord input{}; + TensorRecord weight{}; + TensorRecord bias{}; + TensorRecord output{}; + if (op.num_inputs != 3 || op.num_outputs != 1 || + !read_tensor(graph, op.inputs[0], input) || + !read_tensor(graph, op.inputs[1], weight) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], output) || + !is_int8_activation(input) || !is_int8_activation(output) || + !is_constant_of(weight, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || input.rank != 4 || + output.rank != 4 || weight.rank != 4 || bias.rank != 1 || + input.shape[0] != 1 || output.shape[0] != 1 || + weight.shape[0] != output.shape[1] || weight.shape[3] != input.shape[1] || + bias.shape[0] != output.shape[1]) + return false; + const size_t channels = static_cast(output.shape[1]); + if (op.parameter_size != TRANSPOSE_CONV_PARAMS_SIZE + channels * 8U) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + int32_t f[13]; + for (int k = 0; k < 13; ++k) + f[k] = read_i32(p + 4 * k); + const int32_t kh = f[0], kw = f[1], sh = f[2], sw = f[3], ph = f[4], + pw = f[5]; + const int32_t oph = f[6], opw = f[7], dh = f[8], dw = f[9], groups = f[10]; + if (kh != weight.shape[1] || kw != weight.shape[2] || sh <= 0 || sw <= 0 || + ph < 0 || pw < 0 || oph < 0 || opw < 0 || dh <= 0 || dw <= 0 || + groups != 1 || !valid_activation_range(f[11], f[12])) + return false; + const int64_t expected_h = static_cast(input.shape[2] - 1) * sh - + 2 * ph + static_cast(dh) * (kh - 1) + oph + 1; + const int64_t expected_w = static_cast(input.shape[3] - 1) * sw - + 2 * pw + static_cast(dw) * (kw - 1) + opw + 1; + if (output.shape[2] != expected_h || output.shape[3] != expected_w) + return false; + const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + const uint8_t* shifts = multipliers + channels * 4U; + for (size_t c = 0; c < channels; ++c) { + if (!valid_requantization( + read_i32(multipliers + 4 * c), read_i32(shifts + 4 * c))) + return false; + } + return true; +} + +bool validate_int8_operation( + const GraphView& graph, + const OperationRecord& op) { + TensorRecord a{}; + TensorRecord b{}; + TensorRecord out{}; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + switch (static_cast(op.opcode)) { + case OpCode::CONV2D: + return validate_int8_conv(graph, op, false); + case OpCode::DEPTHWISE_CONV2D: + return validate_int8_conv(graph, op, true); + case OpCode::RELU6: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_PAIR_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_shape(a, out) && + same_quantization(a, out) && + valid_activation_range(read_i32(p), read_i32(p + 4)); + case OpCode::ADD: { + Int8AddParams params{}; + return op.num_inputs == 2 && op.num_outputs == 1 && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.inputs[1], b) && + read_tensor(graph, op.outputs[0], out) && is_int8_value(a) && + is_int8_value(b) && is_int8_activation(out) && same_shape(a, b) && + same_shape(a, out) && read_int8_add_params(graph, op, params) && + params.input1_mult > 0 && params.input2_mult > 0 && + params.output_mult > 0 && params.input1_shift <= 0 && + params.input1_shift >= -31 && params.input2_shift <= 0 && + params.input2_shift >= -31 && params.output_shift <= 0 && + params.output_shift >= -31 && params.left_shift >= 0 && + params.left_shift <= 22 && + valid_activation_range(params.activation_min, params.activation_max); + } + case OpCode::AVGPOOL2D: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_PAIR_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && a.rank == 4 && out.rank == 4 && + a.shape[0] == 1 && out.shape[0] == 1 && a.shape[1] == out.shape[1] && + out.shape[2] == 1 && out.shape[3] == 1 && + static_cast(a.shape[2]) * a.shape[3] * 255 < INT32_MAX && + valid_requantization(read_i32(p), read_i32(p + 4)); + case OpCode::FLATTEN: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == 0 && read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_quantization(a, out) && + out.rank == 2 && out.shape[0] == 1 && + (a.rank != 4 || a.shape[0] == 1) && + tensor_element_count(a) == tensor_element_count(out); + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: { + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size != INT8_POOL_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_quantization(a, out) || + a.rank != 4 || out.rank != 4 || a.shape[0] != 1 || + out.shape[0] != 1 || a.shape[1] != out.shape[1] || + a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535) + return false; + const int32_t kh = p[0]; + const int32_t kw = p[1]; + const int32_t sh = p[2]; + const int32_t sw = p[3]; + const int32_t ph = p[4]; + const int32_t pw = p[5]; + if (kh == 0 || kw == 0 || sh == 0 || sw == 0 || p[6] != 1 || p[7] != 1 || + 2 * ph > kh || 2 * pw > kw || + !valid_activation_range(read_i32(p + 8), read_i32(p + 12))) + return false; + return out.shape[2] == (a.shape[2] + 2 * ph - kh) / sh + 1 && + out.shape[3] == (a.shape[3] + 2 * pw - kw) / sw + 1; + } + case OpCode::LUT: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_LUT_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_shape(a, out); + case OpCode::MUL: { + if (op.num_inputs != 2 || op.num_outputs != 1 || + op.parameter_size != INT8_MUL_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.inputs[1], b) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_value(a) || + !is_int8_value(b) || !is_int8_activation(out) || + !same_shape(a, out) || + !valid_requantization(read_i32(p), read_i32(p + 4)) || + !valid_activation_range(read_i32(p + 8), read_i32(p + 12))) + return false; + const int32_t mode = read_i32(p + 16); + if (mode == MUL_MODE_ELEMENTWISE) + return same_shape(a, b); + return mode == MUL_MODE_CHANNEL_BROADCAST && a.rank == 4 && b.rank == 4 && + a.shape[0] == 1 && b.shape[0] == 1 && b.shape[1] == a.shape[1] && + b.shape[2] == 1 && b.shape[3] == 1; + } + case OpCode::SOFTMAX: { + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size != INT8_SOFTMAX_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_shape(a, out) || + (a.rank != 2 && a.rank != 4) || a.shape[0] != 1 || + out.quant_zero_point != SOFTMAX_OUTPUT_ZERO_POINT || + std::fabs(out.quant_scale - 1.0f / 256.0f) > 1e-9f) + return false; + uint32_t previous = read_u32(p); + if (previous == 0 || previous > (1U << 24)) + return false; + for (size_t k = 1; k < 256; ++k) { + const uint32_t value = read_u32(p + 4 * k); + if (value > previous) + return false; + previous = value; + } + return true; + } + case OpCode::CONCAT: { + if (op.num_inputs < 2 || op.num_outputs != 1 || op.parameter_size != 0 || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(out) || + (out.rank != 2 && out.rank != 4) || out.shape[0] != 1) + return false; + int64_t channels = 0; + for (uint8_t j = 0; j < op.num_inputs; ++j) { + if (!read_tensor(graph, op.inputs[j], a) || !is_int8_activation(a) || + !same_quantization(a, out) || a.rank != out.rank || a.shape[0] != 1) + return false; + for (uint8_t d = 2; d < out.rank; ++d) { + if (a.shape[d] != out.shape[d]) + return false; + } + channels += a.shape[1]; + } + return channels == out.shape[1]; + } + case OpCode::LINEAR: { + TensorRecord bias{}; + if (op.num_inputs != 3 || op.num_outputs != 1 || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.inputs[1], b) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !is_constant_of(b, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || b.rank != 2 || + bias.rank != 1 || bias.shape[0] != b.shape[0] || b.shape[0] > 65535 || + b.shape[1] > 65535 || a.rank == 4 || out.rank == 4 || + tensor_element_count(a) % static_cast(b.shape[1]) != 0 || + tensor_element_count(out) != + tensor_element_count(a) / static_cast(b.shape[1]) * + static_cast(b.shape[0]) || + op.parameter_size != + INT8_PAIR_PARAMS_SIZE + static_cast(b.shape[0]) * 8U || + !valid_activation_range(read_i32(p), read_i32(p + 4))) + return false; + const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; + const uint8_t* shifts = + multipliers + static_cast(b.shape[0]) * 4U; + for (int32_t i = 0; i < b.shape[0]; ++i) { + if (!valid_requantization( + read_i32(multipliers + i * 4), read_i32(shifts + i * 4))) + return false; + } + return true; + } + case OpCode::COPY: + return validate_copy(graph, op); + case OpCode::UPSAMPLE: + return validate_upsample(graph, op); + case OpCode::TRANSPOSE_CONV2D: + return validate_transposed_conv(graph, op); + } + return false; +} + +bool validate_int8_dataflow(const GraphView& graph) { + bool produced[MAX_GRAPH_TENSORS]{}; + bool copy_produced[MAX_GRAPH_TENSORS]{}; + uint32_t input_count = 0; + uint32_t output_count = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0) { + produced[i] = true; + ++input_count; + } + if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) + ++output_count; + } + if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + bool listed_input[MAX_GRAPH_TENSORS]{}; + bool listed_output[MAX_GRAPH_TENSORS]{}; + for (uint32_t i = 0; i < graph.num_inputs; ++i) { + const uint32_t id = graph_input_id(graph, i); + TensorRecord tensor{}; + if (!read_tensor(graph, id, tensor) || + (tensor.flags & TENSOR_FLAG_GRAPH_INPUT) == 0) { + std::printf( + "ERROR: graph IO table lists input %u without the input flag\n", + static_cast(id)); + return false; + } + listed_input[id] = true; + } + for (uint32_t i = 0; i < graph.num_outputs; ++i) { + const uint32_t id = graph_output_id(graph, i); + TensorRecord tensor{}; + if (!read_tensor(graph, id, tensor) || + (tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) == 0) { + std::printf( + "ERROR: graph IO table lists output %u without the output flag\n", + static_cast(id)); + return false; + } + listed_output[id] = true; + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if (((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0 && !listed_input[i]) || + ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && + !listed_output[i])) { + std::printf( + "ERROR: tensor %u is flagged as graph IO but missing from the IO table\n", + static_cast(i)); + return false; + } + } + } + if (input_count == 0 || output_count == 0) { + std::printf( + "ERROR: int8 graph must declare at least one input and one output\n"); + return false; + } + for (uint32_t i = 0; i < graph.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph, i, op); + for (uint8_t j = 0; j < op.num_inputs; ++j) { + TensorRecord tensor{}; + read_tensor(graph, op.inputs[j], tensor); + if ((tensor.flags & TENSOR_FLAG_CONSTANT) == 0 && + !produced[op.inputs[j]]) { + std::printf( + "ERROR: operation %u reads tensor %u before it is produced\n", + static_cast(i), + static_cast(op.inputs[j])); + return false; + } + } + const bool is_copy = static_cast(op.opcode) == OpCode::COPY; + for (uint8_t j = 0; j < op.num_outputs; ++j) { + const uint32_t id = op.outputs[j]; + if (produced[id] && !(is_copy && copy_produced[id])) { + std::printf( + "ERROR: tensor %u has more than one producer\n", + static_cast(id)); + return false; + } + produced[id] = true; + copy_produced[id] = is_copy; + } + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && !produced[i]) { + std::printf( + "ERROR: graph output tensor %u is never produced\n", + static_cast(i)); + return false; + } + } + return true; +} + +} // namespace + +uint32_t read_le_u32(const uint8_t* p) { + return read_u32(p); +} + +int32_t read_le_i32(const uint8_t* p) { + return read_i32(p); +} + +bool is_int8_graph_version(uint32_t version) { + return version == GRAPH_ESP_NN_INT8_VERSION || + version == GRAPH_ESP_NN_INT8_IO_VERSION; +} + +namespace { + +uint32_t flagged_tensor(const GraphView& graph, uint8_t flag, uint32_t index) { + uint32_t seen = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & flag) == 0) + continue; + if (seen == index) + return i; + ++seen; + } + return UINT32_MAX; +} + +uint32_t flagged_count(const GraphView& graph, uint8_t flag) { + uint32_t count = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & flag) != 0) + ++count; + } + return count; +} + +} // namespace + +uint32_t graph_input_count(const GraphView& graph) { + if (graph.io_ids != nullptr) + return graph.num_inputs; + return flagged_count(graph, TENSOR_FLAG_GRAPH_INPUT); +} + +uint32_t graph_output_count(const GraphView& graph) { + if (graph.io_ids != nullptr) + return graph.num_outputs; + return flagged_count(graph, TENSOR_FLAG_GRAPH_OUTPUT); +} + +uint32_t graph_input_id(const GraphView& graph, uint32_t index) { + if (graph.io_ids != nullptr) { + return index < graph.num_inputs ? read_u32(graph.io_ids + 4 * index) + : UINT32_MAX; + } + return flagged_tensor(graph, TENSOR_FLAG_GRAPH_INPUT, index); +} + +uint32_t graph_output_id(const GraphView& graph, uint32_t index) { + if (graph.io_ids != nullptr) { + return index < graph.num_outputs + ? read_u32(graph.io_ids + 4 * (graph.num_inputs + index)) + : UINT32_MAX; + } + return flagged_tensor(graph, TENSOR_FLAG_GRAPH_OUTPUT, index); +} + +bool read_int8_conv_params( + const GraphView& graph, + const OperationRecord& op, + uint32_t channels, + Int8ConvParams& params) { + const size_t expected = + INT8_CONV_BASE_PARAMS_SIZE + static_cast(channels) * 8U; + if (op.parameter_size != expected) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + params.kernel_h = p[0]; + params.kernel_w = p[1]; + params.stride_h = p[2]; + params.stride_w = p[3]; + params.pad_h = p[4]; + params.pad_w = p[5]; + params.dilation_h = p[6]; + params.dilation_w = p[7]; + params.groups = read_i32(p + 8); + params.activation_min = read_i32(p + 12); + params.activation_max = read_i32(p + 16); + params.multipliers = p + INT8_CONV_BASE_PARAMS_SIZE; + params.shifts = params.multipliers + static_cast(channels) * 4U; + return true; +} + +bool read_int8_add_params( + const GraphView& graph, + const OperationRecord& op, + Int8AddParams& params) { + if (op.parameter_size != INT8_ADD_PARAMS_SIZE) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + params.input1_mult = read_i32(p); + params.input1_shift = read_i32(p + 4); + params.input2_mult = read_i32(p + 8); + params.input2_shift = read_i32(p + 12); + params.left_shift = read_i32(p + 16); + params.output_mult = read_i32(p + 20); + params.output_shift = read_i32(p + 24); + params.activation_min = read_i32(p + 28); + params.activation_max = read_i32(p + 32); + return true; +} + +size_t tensor_element_count(const TensorRecord& tensor) { + size_t count = 1; + for (uint8_t i = 0; i < tensor.rank; ++i) { + if (tensor.shape[i] <= 0 || + static_cast(tensor.shape[i]) > SIZE_MAX / count) { + return 0; + } + count *= static_cast(tensor.shape[i]); + } + return count; +} + +size_t tensor_data_size(const TensorRecord& tensor) { + const size_t count = tensor_element_count(tensor); + size_t element_size = 0; + switch (static_cast(tensor.dtype)) { + case DataType::FLOAT32: + case DataType::INT32: + element_size = 4; + break; + case DataType::INT8: + case DataType::UINT8: + element_size = 1; + break; + default: + return 0; + } + if (count == 0 || count > SIZE_MAX / element_size) { + return 0; + } + return count * element_size; +} + +bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor) { + if (index >= graph.header.num_tensors) + return false; + const uint8_t* p = graph.data + graph.tensor_table_offset + + static_cast(index) * graph.tensor_record_size; + tensor.tensor_id = read_u32(p); + tensor.constant_offset = read_u32(p + 4); + tensor.dtype = p[8]; + tensor.flags = p[9]; + tensor.rank = p[11] == 0 ? p[10] : 0xFF; + for (size_t i = 0; i < 4; ++i) + tensor.shape[i] = read_i32(p + 12 + 4 * i); + const uint32_t scale_bits = read_u32(p + 28); + std::memcpy(&tensor.quant_scale, &scale_bits, sizeof(scale_bits)); + tensor.quant_zero_point = read_i32(p + 32); + return true; +} + +bool read_operation( + const GraphView& graph, + uint32_t index, + OperationRecord& operation) { + if (index >= graph.header.num_operations) + return false; + const uint8_t* p = graph.data + graph.operation_table_offset + + static_cast(index) * OPERATION_RECORD_SIZE; + operation.op_id = p[0]; + operation.opcode = p[1]; + operation.num_inputs = p[2]; + operation.num_outputs = p[3]; + for (size_t i = 0; i < 6; ++i) + operation.inputs[i] = read_u32(p + 4 + 4 * i); + for (size_t i = 0; i < 2; ++i) + operation.outputs[i] = read_u32(p + 28 + 4 * i); + operation.parameter_offset = read_u32(p + 36); + operation.parameter_size = read_u32(p + 40); + return true; +} + +const uint8_t* tensor_constant_data( + const GraphView& graph, + const TensorRecord& tensor) { + return graph.constant_data + tensor.constant_offset; +} + +bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { + graph = {}; + if (data == nullptr || size < GRAPH_HEADER_SIZE) { + std::printf("ERROR: graph is null or shorter than the 22-byte header\n"); + return false; + } + if (std::memcmp(data, "XTS3", 4) != 0) { + std::printf("ERROR: invalid graph magic\n"); + return false; + } + graph.header.magic[0] = 'X'; + graph.header.magic[1] = 'T'; + graph.header.magic[2] = 'S'; + graph.header.magic[3] = '3'; + graph.header.version = read_u16(data + 4); + graph.header.num_tensors = read_u32(data + 6); + graph.header.num_operations = read_u32(data + 10); + graph.header.parameter_size = read_u32(data + 14); + graph.header.constant_size = read_u32(data + 18); + if (!is_int8_graph_version(graph.header.version) || + graph.header.num_tensors == 0 || + graph.header.num_tensors > MAX_GRAPH_TENSORS || + graph.header.num_operations == 0 || + graph.header.num_operations > MAX_GRAPH_OPERATIONS) { + std::printf("ERROR: unsupported version or graph record counts\n"); + return false; + } + + graph.data = data; + graph.size = size; + graph.tensor_table_offset = GRAPH_HEADER_SIZE; + if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (!range_valid(GRAPH_HEADER_SIZE, GRAPH_IO_TABLE_HEADER_SIZE, size)) + return false; + graph.num_inputs = read_u16(data + GRAPH_HEADER_SIZE); + graph.num_outputs = read_u16(data + GRAPH_HEADER_SIZE + 2); + const size_t io_bytes = + 4U * (static_cast(graph.num_inputs) + graph.num_outputs); + if (graph.num_inputs == 0 || graph.num_outputs == 0 || + graph.num_inputs > MAX_GRAPH_IO || graph.num_outputs > MAX_GRAPH_IO || + !range_valid( + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE, io_bytes, size)) { + std::printf("ERROR: graph IO table is invalid\n"); + return false; + } + graph.io_ids = data + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE; + for (uint32_t i = 0; + i < static_cast(graph.num_inputs) + graph.num_outputs; + ++i) { + if (read_u32(graph.io_ids + 4 * i) >= graph.header.num_tensors) + return false; + } + graph.tensor_table_offset = + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE + io_bytes; + } + graph.tensor_record_size = TENSOR_RECORD_SIZE; + const size_t tensor_bytes = + static_cast(graph.header.num_tensors) * graph.tensor_record_size; + if (!range_valid(graph.tensor_table_offset, tensor_bytes, size)) + return false; + graph.operation_table_offset = graph.tensor_table_offset + tensor_bytes; + const size_t operation_bytes = + static_cast(graph.header.num_operations) * OPERATION_RECORD_SIZE; + if (!range_valid(graph.operation_table_offset, operation_bytes, size)) + return false; + graph.parameter_offset = graph.operation_table_offset + operation_bytes; + if (!range_valid(graph.parameter_offset, graph.header.parameter_size, size)) + return false; + graph.constant_offset = graph.parameter_offset + graph.header.parameter_size; + if (!range_valid(graph.constant_offset, graph.header.constant_size, size) || + graph.constant_offset + graph.header.constant_size != size) { + std::printf("ERROR: graph sections exceed blob or leave trailing bytes\n"); + return false; + } + if ((graph.constant_offset % INT8_PARAMETER_ALIGNMENT) != 0) { + std::printf( + "ERROR: int8 constant region is not 16-byte aligned within the blob\n"); + return false; + } + graph.parameter_data = data + graph.parameter_offset; + graph.constant_data = data + graph.constant_offset; + + const unsigned allowed_flags = 0x0FU; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if (tensor.tensor_id != i || !valid_dtype(tensor.dtype) || + tensor.rank > 4 || (tensor.flags & ~allowed_flags) != 0) { + std::printf( + "ERROR: tensor record %u has invalid ID, dtype, rank, or flags\n", + static_cast(i)); + return false; + } + for (uint8_t d = 0; d < tensor.rank; ++d) { + if (tensor.shape[d] <= 0) + return false; + } + for (uint8_t d = tensor.rank; d < 4; ++d) { + if (tensor.shape[d] != 0) + return false; + } + const bool is_constant = (tensor.flags & TENSOR_FLAG_CONSTANT) != 0; + const bool has_quantization = (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; + const uint8_t quantized_dtype = static_cast(DataType::INT8); + if ((has_quantization && + (tensor.dtype != quantized_dtype || + !std::isfinite(tensor.quant_scale) || tensor.quant_scale <= 0.0f || + tensor.quant_zero_point < -128 || tensor.quant_zero_point > 127)) || + (!has_quantization && + (tensor.quant_scale != 0.0f || tensor.quant_zero_point != 0))) { + std::printf( + "ERROR: tensor %u quantization metadata is invalid\n", + static_cast(i)); + return false; + } + const bool quantized_constant_allowed = + graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION && + tensor.dtype == static_cast(DataType::INT8); + if ((is_constant + ? ((tensor.dtype != static_cast(DataType::INT8) && + tensor.dtype != static_cast(DataType::INT32)) || + (has_quantization && !quantized_constant_allowed) || + (tensor.flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != 0) + : (tensor.dtype != static_cast(DataType::INT8) || + !has_quantization))) { + std::printf( + "ERROR: tensor %u is not a supported native int8 tensor\n", + static_cast(i)); + return false; + } + const size_t data_size = tensor_data_size(tensor); + if (data_size == 0 || + (is_constant && + !range_valid( + tensor.constant_offset, data_size, graph.header.constant_size)) || + (!is_constant && tensor.constant_offset != 0)) { + std::printf( + "ERROR: tensor %u constant data range or shape is invalid\n", + static_cast(i)); + return false; + } + } + + size_t expected_parameter_offset = 0; + for (uint32_t i = 0; i < graph.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph, i, op); + if (op.op_id != (i & 0xFFU) || + !valid_opcode(op.opcode, graph.header.version) || op.num_inputs == 0 || + op.num_inputs > 6 || op.num_outputs == 0 || op.num_outputs > 2 || + op.parameter_offset != expected_parameter_offset || + !range_valid( + op.parameter_offset, + op.parameter_size, + graph.header.parameter_size)) { + std::printf( + "ERROR: operation record %u is invalid\n", static_cast(i)); + return false; + } + for (uint8_t j = 0; j < op.num_inputs; ++j) + if (!valid_tensor_reference(graph, op.inputs[j])) + return false; + for (uint8_t j = 0; j < op.num_outputs; ++j) + if (!valid_tensor_reference(graph, op.outputs[j])) + return false; + for (uint8_t j = op.num_inputs; j < 6; ++j) + if (op.inputs[j] != 0) + return false; + for (uint8_t j = op.num_outputs; j < 2; ++j) + if (op.outputs[j] != 0) + return false; + if (!validate_int8_operation(graph, op)) { + std::printf( + "ERROR: operation %u (%s) metadata or parameter block is invalid\n", + static_cast(i), + opcode_name(static_cast(op.opcode))); + return false; + } + expected_parameter_offset += op.parameter_size; + } + const size_t parameter_padding = + graph.header.parameter_size - expected_parameter_offset; + if (expected_parameter_offset > graph.header.parameter_size || + parameter_padding >= INT8_PARAMETER_ALIGNMENT) { + std::printf( + "ERROR: parameter region size does not match operation records\n"); + return false; + } + if (!validate_int8_dataflow(graph)) + return false; + return true; +} + +const char* opcode_name(OpCode opcode) { + switch (opcode) { + case OpCode::CONV2D: + return "CONV2D"; + case OpCode::DEPTHWISE_CONV2D: + return "DEPTHWISE_CONV2D"; + case OpCode::RELU6: + return "RELU6"; + case OpCode::ADD: + return "ADD"; + case OpCode::AVGPOOL2D: + return "AVGPOOL2D"; + case OpCode::LINEAR: + return "LINEAR"; + case OpCode::FLATTEN: + return "FLATTEN"; + case OpCode::MAX_POOL2D: + return "MAX_POOL2D"; + case OpCode::AVG_POOL2D: + return "AVG_POOL2D"; + case OpCode::CONCAT: + return "CONCAT"; + case OpCode::LUT: + return "LUT"; + case OpCode::MUL: + return "MUL"; + case OpCode::SOFTMAX: + return "SOFTMAX"; + case OpCode::COPY: + return "COPY"; + case OpCode::UPSAMPLE: + return "UPSAMPLE"; + case OpCode::TRANSPOSE_CONV2D: + return "TRANSPOSE_CONV2D"; + default: + return "UNKNOWN"; + } +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/lowering.py b/backends/espressif/lowering.py new file mode 100644 index 00000000000..cad9b29514b --- /dev/null +++ b/backends/espressif/lowering.py @@ -0,0 +1,36 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import executorch.kernels.quantized # noqa: F401 registers quantize/dequantize out variants + +from executorch.backends.espressif.partition.espnn_partitioner import ( + DuplicateQuantizationFanoutPass, + EspNNPartitioner, +) +from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower + + +def to_espnn_edge(program, allow_cpu_fallback: bool = True): + """ + Lower a PT2E-quantized torch.export program for the ESP32 ExecuTorch + runtime: int8 regions become ESP-NN delegates, anything else stays in + the program for the portable CPU kernels. Returns (edge_manager, report). + """ + partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback) + edge = to_edge_transform_and_lower( + program, + transform_passes=[DuplicateQuantizationFanoutPass()], + partitioner=[partitioner], + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + return edge, partitioner.report + + +__all__ = ["to_espnn_edge"] diff --git a/backends/espressif/partition/__init__.py b/backends/espressif/partition/__init__.py new file mode 100644 index 00000000000..5af6d98f28c --- /dev/null +++ b/backends/espressif/partition/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .espnn_partitioner import DuplicateQuantizationFanoutPass, EspNNPartitioner + +__all__ = ["DuplicateQuantizationFanoutPass", "EspNNPartitioner"] diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py new file mode 100644 index 00000000000..de9380c6e59 --- /dev/null +++ b/backends/espressif/partition/espnn_partitioner.py @@ -0,0 +1,423 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import Callable, Optional + +import torch + +from executorch.backends.espressif.espnn_preprocess import ( + EspNNBackend, + INT8_FORMAT_SPEC, +) +from executorch.backends.espressif.serialization.int8_ops import ( + consumers, + is_dequantize, + is_movement_terminal, + is_quantize, + MOVEMENT_KINDS, + node_kind, + SPLIT_KINDS, +) +from executorch.backends.espressif.serialization.xts3 import ( + _target_kind, + ExportedTensorResolver, + INT8_PASSTHROUGH_MARKERS, + lower_int8_node, + LUT_KINDS, + TensorRegistry, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec +from executorch.exir.backend.partitioner import ( + DelegationSpec, + Partitioner, + PartitionResult, +) +from executorch.exir.backend.utils import tag_constant_data +from torch.export import ExportedProgram +from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner +from torch.fx.passes.infra.pass_base import PassBase, PassResult +from torch.fx.passes.operator_support import OperatorSupportBase + + +SUPPORTED_KINDS = { + "conv2d", + "convolution", + "conv_transpose2d", + "hardtanh", + "hardtanh_", + "relu", + "relu6", + "add", + "adaptive_avg_pool2d", + "mean_dim", + "max_pool2d", + "max_pool2d_with_indices", + "avg_pool2d", + "cat", + "stack", + "flatten", + "reshape", + "linear", + "identity", + "dropout", + "getitem", + "mul", + "softmax", + "pad", + "upsample_nearest2d", + "upsample_nearest2d_vec", + "upsample_bilinear2d", + "upsample_bilinear2d_vec", + *MOVEMENT_KINDS, + *SPLIT_KINDS, + *LUT_KINDS, +} + +PRESERVED_OPS = [ + torch.ops.aten.linear.default, + torch.ops.aten.hardtanh.default, + torch.ops.aten.max_pool2d.default, + torch.ops.aten.flatten.using_ints, + torch.ops.aten.adaptive_avg_pool2d.default, + torch.ops.aten.avg_pool2d.default, + torch.ops.aten.hardswish.default, + torch.ops.aten.hardsigmoid.default, + torch.ops.aten.silu.default, + torch.ops.aten.upsample_nearest2d.vec, + torch.ops.aten.upsample_bilinear2d.vec, +] + + +def is_supported_int8_node(node) -> bool: + if node.op != "call_function": + return False + text = str(node.target) + if any(marker in text for marker in INT8_PASSTHROUGH_MARKERS): + return True + return _target_kind(node.target) in SUPPORTED_KINDS + + +def _keep_if_quantized(node: torch.fx.Node) -> bool: + """Only quantized instances stay whole; float ones decompose for the CPU.""" + tensor_inputs = [arg for arg in node.args if isinstance(arg, torch.fx.Node)] + if not tensor_inputs or not is_dequantize(tensor_inputs[0]): + return False + if not any(is_quantize(user) for user in node.users) and not any( + _target_kind(user.target) in ("relu", "relu6", "hardtanh", "hardtanh_") + and any(is_quantize(u) for u in user.users) + for user in node.users + if user.op == "call_function" + ): + return False + if _target_kind(node.target) == "linear": + value = tensor_inputs[0].meta.get("val") + if value is None or value.dim() == 4 or value.dim() < 1: + return False + return True + + +def _operator_name(target) -> str: + import re + + match = re.search( + r"([A-Za-z_]+(?:\.[A-Za-z0-9_]+){1,2})(?=[>:\s]|$)", + str(target).replace(" bool: + return node.op == "call_function" and any( + marker in str(node.target) for marker in INT8_PASSTHROUGH_MARKERS + ) + + +def duplicate_quantization_fanout(graph_module: torch.fx.GraphModule) -> int: + """ + Give every consumer its own quantize -> dequantize chain so that a + partition boundary can fall between any quantize and dequantize pair. + """ + graph = graph_module.graph + copies = 0 + for node in list(graph.nodes): + if not is_dequantize(node) or len(node.users) < 2: + continue + for user in list(node.users)[1:]: + with graph.inserting_after(node): + duplicate = graph.node_copy(node) + duplicate.meta = dict(node.meta) + user.replace_input_with(node, duplicate) + copies += 1 + for node in list(graph.nodes): + if not is_quantize(node) or len(node.users) < 2: + continue + for user in list(node.users)[1:]: + with graph.inserting_after(node): + duplicate = graph.node_copy(node) + duplicate.meta = dict(node.meta) + user.replace_input_with(node, duplicate) + copies += 1 + if copies: + graph.lint() + graph_module.recompile() + return copies + + +class DuplicateQuantizationFanoutPass(PassBase): + """Run before partitioning (to_edge_transform_and_lower transform_passes).""" + + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: + return PassResult(graph_module, duplicate_quantization_fanout(graph_module) > 0) + + +class StructuralResolver(ExportedTensorResolver): + """ + ExecuTorch partitions a copy of the program whose constants are + FakeTensors. Support is judged on shapes, arguments, and quantization + parameters, so fake constants are replaced by neutral stand-ins of the + same shape and dtype (scales of one, zero points and weights of zero). + """ + + def resolve(self, node): + from torch._subclasses.fake_tensor import FakeTensor + + value = super().resolve(node) + if not isinstance(value, FakeTensor): + return value + shape = tuple(int(d) for d in value.shape) + if value.dtype.is_floating_point: + return torch.ones(shape, dtype=value.dtype) + return torch.zeros(shape, dtype=value.dtype) + + +def find_supported_nodes(exported_program: ExportedProgram) -> tuple[set, dict]: + graph_module = exported_program.graph_module + registry = TensorRegistry(StructuralResolver(exported_program), graph_module) + supported = set() + reasons = {} + for node in graph_module.graph.nodes: + if node.op != "call_function": + continue + if _is_passthrough(node): + supported.add(node) + continue + if _target_kind(node.target) not in SUPPORTED_KINDS: + reasons[node] = "no int8 lowering for this operator" + continue + fused_before = set(registry.fused_nodes) + try: + registry.extra_operations.clear() + lowered = lower_int8_node(node, registry) + if lowered is not None or node in registry.fused_nodes: + supported.add(node) + else: + reasons[node] = "operator was not lowered" + except Exception as error: + registry.fused_nodes = fused_before + reasons[node] = str(error).splitlines()[0][:160] + registry.extra_operations.clear() + + changed = True + while changed: + changed = False + for node in list(supported): + kind = node_kind(node) + deferred = ( + kind in MOVEMENT_KINDS and not is_movement_terminal(node) + ) or kind in SPLIT_KINDS + if not deferred: + continue + downstream = ( + consumers(node) if kind not in SPLIT_KINDS else list(node.users) + ) + if any(user not in supported for user in downstream): + supported.discard(node) + reasons[node] = "data movement feeds an operator that runs on the CPU" + changed = True + return supported, reasons + + +class _SupportSet(OperatorSupportBase): + def __init__(self, nodes: set): + self.nodes = nodes + + def is_node_supported(self, submodules, node: torch.fx.Node) -> bool: + return node in self.nodes + + +def _is_constant(node, resolver: ExportedTensorResolver) -> bool: + return node.op in ("placeholder", "get_attr") and resolver.resolve(node) is not None + + +def _boundary_is_valid(nodes: set, resolver: ExportedTensorResolver) -> bool: + for node in nodes: + for user in node.users: + if user not in nodes and not (is_quantize(node) or is_dequantize(node)): + return False + for source in node.all_input_nodes: + if source in nodes or _is_constant(source, resolver): + continue + if not (is_quantize(node) or is_dequantize(node)): + return False + return True + + +FUSABLE_ACTIVATION_KINDS = ("relu", "relu6", "hardtanh", "hardtanh_") + + +def _attach_outputs_to_producers(partitions: list[set]) -> None: + """ + A quantize node (and an activation fused into its producer) must live in + the producer's partition so the partition exports int8 or dequantized + values only, never a raw accumulator. + """ + owner = {} + for index, nodes in enumerate(partitions): + for node in nodes: + owner[node] = index + changed = True + while changed: + changed = False + for index, nodes in enumerate(partitions): + for node in list(nodes): + if _is_passthrough(node): + continue + for user in node.users: + attach = is_quantize(user) or ( + user.op == "call_function" + and _target_kind(user.target) in FUSABLE_ACTIVATION_KINDS + and user in owner + ) + if not attach or owner.get(user) == index or user not in owner: + continue + partitions[owner[user]].discard(user) + nodes.add(user) + owner[user] = index + changed = True + + +def _has_compute(nodes: set) -> bool: + return any(not _is_passthrough(node) for node in nodes) + + +class EspNNPartitioner(Partitioner): + """ + Delegates PT2E-quantized regions of a graph to the ESP32 ESP-NN backend. + + With allow_cpu_fallback (the default) operators the int8 backend cannot + lower stay in the ExecuTorch program and run on the CPU through the + portable kernels; partition boundaries carry float or int8 tensors. + With allow_cpu_fallback=False any unsupported operator is an error, which + is what the standalone XTS3 firmware path requires. + """ + + def __init__( + self, + compile_specs: Optional[list[CompileSpec]] = None, + allow_cpu_fallback: bool = True, + ): + specs = list(compile_specs or []) + if not any(spec.key == INT8_FORMAT_SPEC.key for spec in specs): + specs.append(INT8_FORMAT_SPEC) + self.delegation_spec = DelegationSpec(EspNNBackend.__name__, specs) + self.allow_cpu_fallback = allow_cpu_fallback + self.report: dict = {} + + def ops_to_not_decompose( + self, ep: ExportedProgram + ) -> tuple[list[torch._ops.OpOverload], Optional[Callable[[torch.fx.Node], bool]]]: + return PRESERVED_OPS, _keep_if_quantized + + def partition(self, exported_program: ExportedProgram) -> PartitionResult: + from torch._subclasses.fake_tensor import unset_fake_temporarily + + graph_module = exported_program.graph_module + with unset_fake_temporarily(): + supported, reasons = find_supported_nodes(exported_program) + unsupported = [ + node + for node in graph_module.graph.nodes + if node.op == "call_function" and node not in supported + ] + if unsupported and not self.allow_cpu_fallback: + raise RuntimeError( + "The ESP32 int8 backend cannot run these operators: " + + ", ".join( + f"{node.name} ({node.target}): {reasons.get(node, '')}" + for node in unsupported + ) + ) + + with unset_fake_temporarily(): + resolver = StructuralResolver(exported_program) + proposer = CapabilityBasedPartitioner( + graph_module, _SupportSet(supported), allows_single_node_partition=True + ) + partitions = [ + set(partition.nodes) for partition in proposer.propose_partitions() + ] + _attach_outputs_to_producers(partitions) + tags = {} + rejected = 0 + for nodes in partitions: + if not _has_compute(nodes): + continue + with unset_fake_temporarily(): + valid = _boundary_is_valid(nodes, resolver) + if not valid: + rejected += 1 + continue + tag = f"espnn_{len(tags)}" + for node in nodes: + node.meta["delegation_tag"] = tag + tags[tag] = self.delegation_spec + if not tags and not self.allow_cpu_fallback: + raise RuntimeError("No ESP32 int8 partition could be formed") + tag_constant_data(exported_program) + + delegated = { + node + for node in graph_module.graph.nodes + if "delegation_tag" in node.meta and node.op == "call_function" + } + cpu_ops = [ + node + for node in graph_module.graph.nodes + if node.op == "call_function" + and node not in delegated + and not _is_passthrough(node) + ] + self.report = { + "partitions": len(tags), + "rejected_partitions": rejected, + "delegated_ops": sum(1 for node in delegated if not _is_passthrough(node)), + "cpu_ops": [ + { + "name": node.name, + "target": _operator_name(node.target), + "reason": reasons.get(node, "outside the delegated regions"), + } + for node in cpu_ops + if "getitem" not in str(node.target) + ], + } + return PartitionResult( + tagged_exported_program=exported_program, partition_tags=tags + ) + + +__all__ = [ + "DuplicateQuantizationFanoutPass", + "StructuralResolver", + "EspNNPartitioner", + "duplicate_quantization_fanout", + "find_supported_nodes", + "is_supported_int8_node", +] diff --git a/backends/espressif/quantizer/__init__.py b/backends/espressif/quantizer/__init__.py new file mode 100644 index 00000000000..60a6bccfd8d --- /dev/null +++ b/backends/espressif/quantizer/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .espnn_quantizer import EspNNQuantizer + +__all__ = ["EspNNQuantizer"] diff --git a/backends/espressif/quantizer/espnn_quantizer.py b/backends/espressif/quantizer/espnn_quantizer.py new file mode 100644 index 00000000000..0b710faf3f9 --- /dev/null +++ b/backends/espressif/quantizer/espnn_quantizer.py @@ -0,0 +1,372 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import Optional + +import torch + +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( + get_symmetric_quantization_config, + XNNPACKQuantizer, +) +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer_utils import ( + _is_share_obs_or_fq_op, +) +from torch.fx import Node +from torchao.quantization.pt2e.quantizer import ( + FixedQParamsQuantizationSpec, + get_bias_qspec, + get_input_act_qspec, + get_output_act_qspec, + get_weight_qspec, + QuantizationAnnotation, + QuantizationConfig, + SharedQuantizationSpec, +) +from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY + + +CONV_TARGETS = (torch.ops.aten.conv2d.default,) +LINEAR_TARGETS = (torch.ops.aten.linear.default,) +ADD_TARGETS = (torch.ops.aten.add.Tensor, torch.ops.aten.add_.Tensor) +RELU_TARGETS = (torch.ops.aten.relu.default, torch.ops.aten.relu_.default) +HARDTANH_TARGETS = (torch.ops.aten.hardtanh.default, torch.ops.aten.hardtanh_.default) +RELU6_TARGETS = (torch.ops.aten.relu6.default,) +LUT_TARGETS = ( + torch.ops.aten.hardswish.default, + torch.ops.aten.hardswish_.default, + torch.ops.aten.hardsigmoid.default, + torch.ops.aten.hardsigmoid_.default, + torch.ops.aten.sigmoid.default, + torch.ops.aten.sigmoid_.default, + torch.ops.aten.silu.default, + torch.ops.aten.silu_.default, + torch.ops.aten.tanh.default, + torch.ops.aten.tanh_.default, + torch.ops.aten.gelu.default, + torch.ops.aten.elu.default, + torch.ops.aten.elu_.default, + torch.ops.aten.leaky_relu.default, + torch.ops.aten.leaky_relu_.default, +) +SOFTMAX_TARGETS = (torch.ops.aten.softmax.int, torch.ops.aten._softmax.default) +SOFTMAX_OUTPUT_SCALE = 1.0 / 256.0 +SOFTMAX_OUTPUT_ZERO_POINT = -128 + + +def fused_activation_kind(node: Node) -> Optional[str]: + if node.op != "call_function": + return None + if node.target in RELU_TARGETS: + return "relu" + if node.target in RELU6_TARGETS: + return "relu6" + if node.target in HARDTANH_TARGETS: + minimum = float(node.args[1]) if len(node.args) > 1 else -1.0 + maximum = float(node.args[2]) if len(node.args) > 2 else 1.0 + if minimum == 0.0 and maximum == 6.0: + return "relu6" + return None + + +def _is_annotated(node: Node) -> bool: + annotation = node.meta.get(Q_ANNOTATION_KEY) + return annotation is not None and annotation._annotated + + +def _mark_annotated(nodes) -> None: + for node in nodes: + if node is None: + continue + if Q_ANNOTATION_KEY not in node.meta: + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation() + node.meta[Q_ANNOTATION_KEY]._annotated = True + + +def _annotate_weighted_activation( + producer: Node, + activation: Node, + config: QuantizationConfig, +) -> bool: + input_act = producer.args[0] + weight = producer.args[1] + bias = producer.args[2] if len(producer.args) > 2 else None + if not isinstance(input_act, Node) or not isinstance(weight, Node): + return False + nodes = [producer, activation, weight] + if isinstance(bias, Node): + nodes.append(bias) + if any(_is_annotated(node) for node in nodes): + return False + + input_qspec_map = { + input_act: get_input_act_qspec(config), + weight: get_weight_qspec(config), + } + if isinstance(bias, Node): + input_qspec_map[bias] = get_bias_qspec(config) + + producer.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map=input_qspec_map, + _annotated=True, + ) + activation.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + output_qspec=get_output_act_qspec(config), + _annotated=True, + ) + _mark_annotated(nodes) + return True + + +def _annotate_add_activation( + add: Node, + activation: Node, + config: QuantizationConfig, +) -> bool: + lhs = add.args[0] + rhs = add.args[1] + if not isinstance(lhs, Node) or not isinstance(rhs, Node): + return False + if _is_annotated(add) or _is_annotated(activation): + return False + add.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={ + lhs: get_input_act_qspec(config), + rhs: get_input_act_qspec(config), + }, + _annotated=True, + ) + activation.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + output_qspec=get_output_act_qspec(config), + _annotated=True, + ) + return True + + +def annotate_fused_activations( + model: torch.fx.GraphModule, + config: QuantizationConfig, +) -> int: + fused = 0 + for node in model.graph.nodes: + if fused_activation_kind(node) is None: + continue + producer = node.args[0] + if not isinstance(producer, Node) or producer.op != "call_function": + continue + if len(producer.users) != 1: + continue + if producer.target in CONV_TARGETS or producer.target in LINEAR_TARGETS: + fused += int(_annotate_weighted_activation(producer, node, config)) + elif producer.target in ADD_TARGETS: + fused += int(_annotate_add_activation(producer, node, config)) + return fused + + +class EspNNQuantizer(XNNPACKQuantizer): + """ + PT2E quantizer matching the ESP-NN int8 kernel contract. + + Activations: int8 per-tensor affine. + Weights: int8 per-output-channel symmetric. + ReLU and ReLU6 that directly follow conv2d, linear, or add are fused + into the producer so that the clamp happens on the producer's + requantized output, exactly as the ESP-NN activation_min/max arguments + implement it. Without this fusion ReLU6 would inherit the producer's + full-range scale and waste most of the int8 grid. + """ + + def __init__(self, per_channel: bool = True, is_qat: bool = False): + super().__init__() + self.set_global( + get_symmetric_quantization_config(is_per_channel=per_channel, is_qat=is_qat) + ) + self.fused_activation_count = 0 + + def transform_for_annotation( + self, model: torch.fx.GraphModule + ) -> torch.fx.GraphModule: + model = super().transform_for_annotation(model) + self.rewritten_split_count = rewrite_splits_as_slices(model) + return model + + def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: + if self.global_config is not None: + self.fused_activation_count = annotate_fused_activations( + model, self.global_config + ) + model = self._annotate_for_quantization_config(model) + if self.global_config is not None: + annotate_unary_and_softmax(model, self.global_config) + propagate_shared_quantization(model) + return model + + +def annotate_unary_and_softmax( + model: torch.fx.GraphModule, config: QuantizationConfig +) -> None: + softmax_output = FixedQParamsQuantizationSpec( + dtype=torch.int8, + scale=SOFTMAX_OUTPUT_SCALE, + zero_point=SOFTMAX_OUTPUT_ZERO_POINT, + quant_min=-128, + quant_max=127, + qscheme=torch.per_tensor_affine, + ) + for node in model.graph.nodes: + if node.op != "call_function" or _is_annotated(node): + continue + if node.target not in LUT_TARGETS and node.target not in SOFTMAX_TARGETS: + continue + source = node.args[0] + if not isinstance(source, Node): + continue + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={source: get_input_act_qspec(config)}, + output_qspec=( + softmax_output + if node.target in SOFTMAX_TARGETS + else get_output_act_qspec(config) + ), + _annotated=True, + ) + + +EXTRA_SHARED_QPARAM_TARGETS = ( + torch.ops.aten.avg_pool2d.default, + torch.ops.aten.relu6.default, + torch.ops.aten.unsqueeze.default, + torch.ops.aten.squeeze.default, + torch.ops.aten.squeeze.dims, + torch.ops.aten.slice.Tensor, + torch.ops.aten.select.int, + torch.ops.aten.expand.default, + torch.ops.aten.unflatten.int, + torch.ops.aten._unsafe_view.default, + torch.ops.aten.clone.default, + torch.ops.aten.contiguous.default, + torch.ops.aten.upsample_nearest2d.vec, + torch.ops.aten.upsample_nearest2d.default, + torch.ops.aten.upsample_bilinear2d.vec, + torch.ops.aten.upsample_bilinear2d.default, + torch.ops.aten.constant_pad_nd.default, + torch.ops.aten.pad.default, +) + +SPLIT_TARGETS = ( + torch.ops.aten.split.Tensor, + torch.ops.aten.split_with_sizes.default, + torch.ops.aten.chunk.default, + torch.ops.aten.unbind.int, +) + + +def rewrite_splits_as_slices(model: torch.fx.GraphModule) -> int: + """ + Replace split/chunk/unbind + getitem with slice/select so that every + piece is an ordinary tensor that can carry a shared quantization spec. + """ + import operator + + rewritten = 0 + for node in list(model.graph.nodes): + if node.op != "call_function" or node.target not in SPLIT_TARGETS: + continue + source = node.args[0] + value = source.meta.get("val") if isinstance(source, Node) else None + if value is None: + continue + users = list(node.users) + if any(user.target is not operator.getitem for user in users): + continue + rank = value.dim() + if node.target is torch.ops.aten.unbind.int: + dim = int(node.args[1]) if len(node.args) > 1 else 0 + dim %= rank + bounds = None + else: + dim = ( + int(node.args[2]) + if len(node.args) > 2 + else int(node.kwargs.get("dim", 0)) + ) + dim %= rank + size = int(value.shape[dim]) + if node.target is torch.ops.aten.chunk.default: + chunks = int(node.args[1]) + step = (size + chunks - 1) // chunks + sizes = [min(step, size - start) for start in range(0, size, step)] + elif isinstance(node.args[1], (list, tuple)): + sizes = [int(v) for v in node.args[1]] + else: + step = int(node.args[1]) + sizes = [min(step, size - start) for start in range(0, size, step)] + starts = [sum(sizes[:i]) for i in range(len(sizes))] + bounds = [(start, start + length) for start, length in zip(starts, sizes)] + for user in users: + index = int(user.args[1]) + with model.graph.inserting_before(user): + if bounds is None: + replacement = model.graph.call_function( + torch.ops.aten.select.int, (source, dim, index) + ) + else: + start, end = bounds[index] + replacement = model.graph.call_function( + torch.ops.aten.slice.Tensor, (source, dim, start, end, 1) + ) + replacement.meta = dict(user.meta) + user.replace_all_uses_with(replacement) + model.graph.erase_node(user) + model.graph.erase_node(node) + rewritten += 1 + if rewritten: + model.graph.lint() + model.recompile() + return rewritten + + +def _shares_qparams(node: Node) -> bool: + return node.op == "call_function" and ( + _is_share_obs_or_fq_op(node.target) + or node.target in EXTRA_SHARED_QPARAM_TARGETS + ) + + +def propagate_shared_quantization(model: torch.fx.GraphModule) -> None: + for node in model.graph.nodes: + if not _shares_qparams(node): + continue + if _is_annotated(node): + continue + source = node.args[0] + if not isinstance(source, Node): + continue + annotation = source.meta.get(Q_ANNOTATION_KEY) + if annotation is None or annotation.output_qspec is None: + continue + shared = SharedQuantizationSpec(source) + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={source: shared}, + output_qspec=shared, + _annotated=True, + ) + + +__all__ = [ + "EspNNQuantizer", + "annotate_fused_activations", + "fused_activation_kind", + "propagate_shared_quantization", + "annotate_unary_and_softmax", + "rewrite_splits_as_slices", + "SOFTMAX_OUTPUT_SCALE", + "SOFTMAX_OUTPUT_ZERO_POINT", +] diff --git a/backends/espressif/reference/__init__.py b/backends/espressif/reference/__init__.py new file mode 100644 index 00000000000..79d9f63d14a --- /dev/null +++ b/backends/espressif/reference/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .esp_nn_reference import ReferenceInterpreter +from .program_emulator import ProgramEmulator + +__all__ = ["ProgramEmulator", "ReferenceInterpreter"] diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py new file mode 100644 index 00000000000..291ccca80f7 --- /dev/null +++ b/backends/espressif/reference/esp_nn_reference.py @@ -0,0 +1,643 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +import numpy as np + +from ..serialization.xts3 import ( + COPY_FLAG_FILL, + COPY_HEADER, + COPY_HEADER_SIZE, + COPY_MAX_RANK, + COPY_SEGMENT, + COPY_SEGMENT_SIZE, + INT8_ADD_PARAMS, + INT8_CONV_BASE_PARAMS, + INT8_CONV_BASE_PARAMS_SIZE, + INT8_MUL_PARAMS, + INT8_PAIR_PARAMS, + INT8_PAIR_PARAMS_SIZE, + INT8_POOL_PARAMS, + MUL_MODE_CHANNEL_BROADCAST, + SOFTMAX_OUTPUT_ZERO_POINT, + TENSOR_FLAG_GRAPH_INPUT, + TENSOR_FLAG_GRAPH_OUTPUT, + TRANSPOSE_CONV_PARAMS, + TRANSPOSE_CONV_PARAMS_SIZE, + UPSAMPLE_HEADER, + UPSAMPLE_HEADER_SIZE, + UPSAMPLE_WEIGHT_BITS, + XTS3_VERSIONS, + XTS3DType, + XTS3Graph, + XTS3OpCode, + XTS3Tensor, +) + + +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 + + +def wrap_int32(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=np.int64) + return ((values + (1 << 31)) & 0xFFFFFFFF) - (1 << 31) + + +def saturating_rounding_doubling_high_mul(a, b) -> np.ndarray: + a = np.asarray(a, dtype=np.int64) + b = np.asarray(b, dtype=np.int64) + overflow = (a == b) & (a == INT32_MIN) + nudge = np.where((a < 0) ^ (b < 0), 1 - (1 << 30), 1 << 30) + product = a * b + nudge + high = np.where(product >= 0, product >> 31, -((-product) >> 31)) + return np.where(overflow, INT32_MAX, high) + + +def rounding_divide_by_power_of_two(x, exponent) -> np.ndarray: + x = np.asarray(x, dtype=np.int64) + exponent = np.asarray(exponent, dtype=np.int64) + mask = (np.int64(1) << exponent) - 1 + remainder = x & mask + result = x >> exponent + threshold = (mask >> 1) + (result < 0) + return result + (remainder > threshold) + + +def multiply_by_quantized_multiplier(x, multiplier, shift) -> np.ndarray: + shift = np.asarray(shift, dtype=np.int64) + multiplier = np.asarray(multiplier, dtype=np.int64) + left = np.maximum(shift, 0) + right = np.maximum(-shift, 0) + shifted = wrap_int32(np.asarray(x, dtype=np.int64) << left) + return rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(shifted, multiplier), right + ) + + +def _exact_matmul(a: np.ndarray, b: np.ndarray) -> np.ndarray: + result = a.astype(np.float64) @ b.astype(np.float64) + if result.size and np.max(np.abs(result)) >= float(1 << 52): + raise OverflowError("Accumulator exceeds the exactly representable range") + return np.rint(result).astype(np.int64) + + +def device_shape(tensor: XTS3Tensor) -> tuple[int, ...]: + shape = tuple(tensor.shape) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError( + f"Tensor {tensor.tensor_id} has batch {shape[0]}; only batch 1 is supported" + ) + return (shape[2], shape[3], shape[1]) + if len(shape) == 2 and shape[0] == 1: + return (shape[1],) + return shape + + +def to_device_layout(tensor: XTS3Tensor, logical: np.ndarray) -> np.ndarray: + logical = np.asarray(logical) + if logical.shape != tuple(tensor.shape): + raise ValueError(f"Expected shape {tensor.shape}, got {logical.shape}") + if len(tensor.shape) == 4: + return np.ascontiguousarray(logical[0].transpose(1, 2, 0)) + return np.ascontiguousarray(logical.reshape(device_shape(tensor))) + + +def to_logical_layout(tensor: XTS3Tensor, device: np.ndarray) -> np.ndarray: + if len(tensor.shape) == 4: + return np.ascontiguousarray(device.transpose(2, 0, 1)[None]) + return np.ascontiguousarray(device.reshape(tensor.shape)) + + +def quantize(tensor: XTS3Tensor, values: np.ndarray) -> np.ndarray: + inverse = np.float32(1.0 / tensor.quant_scale) + scaled = np.asarray(values, dtype=np.float32) * inverse + quantized = np.rint(scaled).astype(np.int64) + tensor.quant_zero_point + return np.clip(quantized, -128, 127).astype(np.int8) + + +def dequantize(tensor: XTS3Tensor, values: np.ndarray) -> np.ndarray: + return ( + values.astype(np.float32) - np.float32(tensor.quant_zero_point) + ) * np.float32(tensor.quant_scale) + + +@dataclass +class ConvParams: + kernel_h: int + kernel_w: int + stride_h: int + stride_w: int + pad_h: int + pad_w: int + groups: int + act_min: int + act_max: int + multipliers: np.ndarray + shifts: np.ndarray + dilation_h: int = 1 + dilation_w: int = 1 + + +def requantize(values, multiplier, shift, in_zero_point, out_zero_point) -> np.ndarray: + shifted = np.asarray(values, dtype=np.int64) - in_zero_point + result = ( + multiply_by_quantized_multiplier(shifted, multiplier, shift) + out_zero_point + ) + return np.clip(result, -128, 127) + + +def run_copy( + params: bytes, sources: list[np.ndarray], output: np.ndarray +) -> np.ndarray: + count, flags, fill = struct.unpack_from(COPY_HEADER, params, 0) + flat = output.reshape(-1).copy() + if flags & COPY_FLAG_FILL: + flat[:] = fill + offset = COPY_HEADER_SIZE + for _ in range(count): + fields = struct.unpack_from(COPY_SEGMENT, params, offset) + offset += COPY_SEGMENT_SIZE + index, rank, in_offset, out_offset = fields[:4] + dims = fields[4 : 4 + COPY_MAX_RANK][:rank] + in_strides = fields[4 + COPY_MAX_RANK : 4 + 2 * COPY_MAX_RANK][:rank] + out_strides = fields[4 + 2 * COPY_MAX_RANK : 4 + 3 * COPY_MAX_RANK][:rank] + requant, multiplier, shift, in_zp, out_zp = fields[4 + 3 * COPY_MAX_RANK :] + grids = np.meshgrid( + *[np.arange(d, dtype=np.int64) for d in dims], indexing="ij" + ) + in_index = np.full(grids[0].shape, in_offset, dtype=np.int64) + out_index = np.full(grids[0].shape, out_offset, dtype=np.int64) + for grid, in_stride, out_stride in zip(grids, in_strides, out_strides): + in_index += grid * in_stride + out_index += grid * out_stride + values = sources[index].reshape(-1)[in_index.reshape(-1)].astype(np.int64) + if requant: + values = requantize(values, multiplier, shift, in_zp, out_zp) + flat[out_index.reshape(-1)] = values + return flat.reshape(output.shape) + + +def parse_conv_params(params: bytes, channels: int) -> ConvParams: + expected = INT8_CONV_BASE_PARAMS_SIZE + 8 * channels + if len(params) != expected: + raise ValueError( + f"Convolution parameters are {len(params)} bytes, expected {expected}" + ) + kh, kw, sh, sw, ph, pw, dh, dw, groups, act_min, act_max = struct.unpack_from( + INT8_CONV_BASE_PARAMS, params, 0 + ) + base = INT8_CONV_BASE_PARAMS_SIZE + multipliers = np.frombuffer( + params, dtype=" np.ndarray: + tensor = self.graph.tensors[tensor_id] + dtype = {XTS3DType.INT8: np.int8, XTS3DType.INT32: " np.ndarray: + if tensor_id in values: + return values[tensor_id] + tensor = self.graph.tensors[tensor_id] + if tensor.is_constant: + return ( + self.constant(tensor_id).astype(np.int8).reshape(device_shape(tensor)) + ) + raise KeyError(f"Tensor {tensor_id} has not been produced") + + def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: + values: dict[int, np.ndarray] = {} + for tensor_id in self.inputs: + tensor = self.graph.tensors[tensor_id] + data = np.asarray(inputs[tensor_id], dtype=np.int8) + if data.shape != device_shape(tensor): + raise ValueError( + f"Input {tensor_id} must have device shape {device_shape(tensor)}" + ) + values[tensor_id] = data + for operation in self.graph.operations: + values[operation.outputs[0]] = self._run_operation(operation, values) + return values + + def run_outputs(self, inputs: list[np.ndarray]) -> list[np.ndarray]: + values = self.run(dict(zip(self.inputs, inputs))) + return [values[tensor_id] for tensor_id in self.outputs] + + def _run_operation(self, operation, values) -> np.ndarray: # noqa: C901 + tensors = self.graph.tensors + output = tensors[operation.outputs[0]] + opcode = operation.opcode + + if opcode in (XTS3OpCode.CONV2D, XTS3OpCode.DEPTHWISE_CONV2D): + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) + channels = output.shape[1] + p = parse_conv_params(operation.params, channels) + weights = self.constant(operation.inputs[1]) + bias = self.constant(operation.inputs[2]) + height, width, in_channels = x.shape + out_h, out_w = output.shape[2], output.shape[3] + dh, dw = p.dilation_h, p.dilation_w + padded = np.zeros( + (height + 2 * p.pad_h, width + 2 * p.pad_w, in_channels), dtype=np.int64 + ) + padded[p.pad_h : p.pad_h + height, p.pad_w : p.pad_w + width] = ( + x - source.quant_zero_point + ) + accumulator = np.zeros((out_h, out_w, channels), dtype=np.int64) + if opcode == XTS3OpCode.CONV2D: + groups = p.groups + group_in = in_channels // groups + group_out = channels // groups + filters = weights.reshape(channels, p.kernel_h, p.kernel_w, group_in) + for kh in range(p.kernel_h): + for kw in range(p.kernel_w): + patch = padded[ + kh * dh : kh * dh + + p.stride_h * (out_h - 1) + + 1 : p.stride_h, + kw * dw : kw * dw + + p.stride_w * (out_w - 1) + + 1 : p.stride_w, + ] + for g in range(groups): + accumulator[ + :, :, g * group_out : (g + 1) * group_out + ] += _exact_matmul( + patch[:, :, g * group_in : (g + 1) * group_in].reshape( + -1, group_in + ), + filters[ + g * group_out : (g + 1) * group_out, kh, kw, : + ].T, + ).reshape( + out_h, out_w, group_out + ) + else: + multiplier = channels // in_channels + filters = weights.reshape(p.kernel_h, p.kernel_w, channels) + expanded = ( + np.repeat(padded, multiplier, axis=2) if multiplier > 1 else padded + ) + for kh in range(p.kernel_h): + for kw in range(p.kernel_w): + patch = expanded[ + kh * dh : kh * dh + + p.stride_h * (out_h - 1) + + 1 : p.stride_h, + kw * dw : kw * dw + + p.stride_w * (out_w - 1) + + 1 : p.stride_w, + ] + accumulator += patch * filters[kh, kw] + accumulator = wrap_int32(accumulator + bias) + result = multiply_by_quantized_multiplier( + accumulator, p.multipliers, p.shifts + ) + result = result + output.quant_zero_point + return np.clip(result, p.act_min, p.act_max).astype(np.int8) + + if opcode == XTS3OpCode.RELU6: + act_min, act_max = struct.unpack(INT8_PAIR_PARAMS, operation.params) + return np.clip(values[operation.inputs[0]], act_min, act_max).astype( + np.int8 + ) + + if opcode == XTS3OpCode.ADD: + ( + lhs_mult, + lhs_shift, + rhs_mult, + rhs_shift, + left_shift, + out_mult, + out_shift, + act_min, + act_max, + ) = struct.unpack(INT8_ADD_PARAMS, operation.params) + lhs = tensors[operation.inputs[0]] + rhs = tensors[operation.inputs[1]] + a = ( + self.value(lhs.tensor_id, values).astype(np.int64) + - lhs.quant_zero_point + ) << left_shift + b = ( + self.value(rhs.tensor_id, values).astype(np.int64) + - rhs.quant_zero_point + ) << left_shift + a = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(a, lhs_mult), -lhs_shift + ) + b = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(b, rhs_mult), -rhs_shift + ) + total = wrap_int32(a + b) + total = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(total, out_mult), -out_shift + ) + total = total + output.quant_zero_point + return np.clip(total, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.AVGPOOL2D: + multiplier, shift = struct.unpack(INT8_PAIR_PARAMS, operation.params) + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) + elements = x.shape[0] * x.shape[1] + total = x.sum(axis=(0, 1)) - elements * source.quant_zero_point + result = multiply_by_quantized_multiplier(total, multiplier, shift) + result = result + output.quant_zero_point + return np.clip(result, -128, 127).astype(np.int8).reshape(1, 1, -1) + + if opcode == XTS3OpCode.FLATTEN: + source = tensors[operation.inputs[0]] + x = values[source.tensor_id] + if len(source.shape) == 4: + return np.ascontiguousarray(x.transpose(2, 0, 1)).reshape(-1) + return x.reshape(-1) + + if opcode in (XTS3OpCode.MAX_POOL2D, XTS3OpCode.AVG_POOL2D): + kh, kw, sh, sw, ph, pw, _, _, act_min, act_max = struct.unpack( + INT8_POOL_PARAMS, operation.params + ) + x = values[operation.inputs[0]].astype(np.int64) + height, width, channels = x.shape + out_h, out_w = output.shape[2], output.shape[3] + result = np.zeros((out_h, out_w, channels), dtype=np.int64) + for oy in range(out_h): + y0 = oy * sh - ph + ys, ye = max(0, y0), min(height, y0 + kh) + for ox in range(out_w): + x0 = ox * sw - pw + xs, xe = max(0, x0), min(width, x0 + kw) + window = x[ys:ye, xs:xe].reshape(-1, channels) + if opcode == XTS3OpCode.MAX_POOL2D: + result[oy, ox] = window.max(axis=0) + else: + count = window.shape[0] + total = window.sum(axis=0) + half = count // 2 + numerator = np.where(total > 0, total + half, total - half) + result[oy, ox] = np.sign(numerator) * ( + np.abs(numerator) // count + ) + return np.clip(result, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.LUT: + table = np.frombuffer(operation.params, dtype=np.int8) + x = values[operation.inputs[0]].astype(np.int64) + return table[x + 128] + + if opcode == XTS3OpCode.MUL: + multiplier, shift, act_min, act_max, mode = struct.unpack( + INT8_MUL_PARAMS, operation.params + ) + lhs = tensors[operation.inputs[0]] + rhs = tensors[operation.inputs[1]] + a = ( + self.value(lhs.tensor_id, values).astype(np.int64) + - lhs.quant_zero_point + ) + b = ( + self.value(rhs.tensor_id, values).astype(np.int64) + - rhs.quant_zero_point + ) + if mode == MUL_MODE_CHANNEL_BROADCAST: + b = b.reshape(-1) + product = multiply_by_quantized_multiplier(a * b, multiplier, shift) + product = product + output.quant_zero_point + return np.clip(product, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.SOFTMAX: + table = np.frombuffer(operation.params, dtype="> ( + 2 * UPSAMPLE_WEIGHT_BITS + ) + return np.clip(result, -128, 127).astype(np.int8) + + if opcode == XTS3OpCode.TRANSPOSE_CONV2D: + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) - source.quant_zero_point + channels = output.shape[1] + fields = struct.unpack_from(TRANSPOSE_CONV_PARAMS, operation.params, 0) + kh, kw, sh, sw, ph, pw, _, _, dh, dw, _, act_min, act_max = fields + base = TRANSPOSE_CONV_PARAMS_SIZE + multipliers = np.frombuffer( + operation.params, dtype="= 0) & (ox < out_w) + if not np.any(valid): + continue + contribution = _exact_matmul( + x[iy][valid], filters[:, ky, kx, :].T + ) + accumulator[oy, ox[valid]] += contribution + accumulator = wrap_int32(accumulator + bias) + result = multiply_by_quantized_multiplier(accumulator, multipliers, shifts) + result = result + output.quant_zero_point + return np.clip(result, act_min, act_max).astype(np.int8) + + raise ValueError(f"Unsupported opcode {opcode}") + + +def plan_activation_arena( + graph: XTS3Graph, alignment: int = 16 +) -> tuple[int, dict[int, int]]: + """Mirror of Int8Interpreter::plan_memory in the firmware.""" + op_count = len(graph.operations) + big = 1 << 30 + first: dict[int, int] = {} + last: dict[int, int] = {} + sizes: dict[int, int] = {} + for tensor in graph.tensors: + if tensor.is_constant: + continue + first[tensor.tensor_id] = -1 if tensor.flags & TENSOR_FLAG_GRAPH_INPUT else big + last[tensor.tensor_id] = ( + op_count if tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT else -1 + ) + size = int(np.prod(tensor.shape)) + sizes[tensor.tensor_id] = (size + alignment - 1) // alignment * alignment + for index, operation in enumerate(graph.operations): + for tensor_id in operation.inputs: + if tensor_id in last: + last[tensor_id] = max(last[tensor_id], index) + for tensor_id in operation.outputs: + first[tensor_id] = min(first[tensor_id], index) + last[tensor_id] = max(last[tensor_id], index) + order = sorted( + (tensor_id for tensor_id in sizes if first[tensor_id] != big), + key=lambda tensor_id: (-sizes[tensor_id], tensor_id), + ) + offsets: dict[int, int] = {} + arena = 0 + for tensor_id in order: + candidate = 0 + moved = True + while moved: + moved = False + for other, offset in offsets.items(): + if not ( + first[other] <= last[tensor_id] and first[tensor_id] <= last[other] + ): + continue + if ( + candidate < offset + sizes[other] + and offset < candidate + sizes[tensor_id] + ): + candidate = offset + sizes[other] + moved = True + offsets[tensor_id] = candidate + arena = max(arena, candidate + sizes[tensor_id]) + return arena, offsets + + +__all__ = [ + "ReferenceInterpreter", + "plan_activation_arena", + "dequantize", + "device_shape", + "multiply_by_quantized_multiplier", + "quantize", + "rounding_divide_by_power_of_two", + "saturating_rounding_doubling_high_mul", + "to_device_layout", + "to_logical_layout", +] diff --git a/backends/espressif/reference/program_emulator.py b/backends/espressif/reference/program_emulator.py new file mode 100644 index 00000000000..98754ab76b2 --- /dev/null +++ b/backends/espressif/reference/program_emulator.py @@ -0,0 +1,98 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import numpy as np +import torch + +from ..serialization.xts3 import parse_graph + +from . import esp_nn_reference as reference + + +def _is_dequantize(node) -> bool: + return node.op == "call_function" and "dequantize_per_tensor" in str(node.target) + + +def delegate_output_dtypes(lowered_module) -> list[torch.dtype]: + program = lowered_module.original_module + output = next(node for node in program.graph.nodes if node.op == "output") + values = output.args[0] + if not isinstance(values, (list, tuple)): + values = (values,) + return [torch.float32 if _is_dequantize(value) else torch.int8 for value in values] + + +class DelegateEmulator: + """Runs one ESP32 int8 delegate payload exactly as the firmware backend does.""" + + def __init__(self, lowered_module): + self.graph = parse_graph(bytes(lowered_module.processed_bytes)) + self.interpreter = reference.ReferenceInterpreter(self.graph) + self.output_dtypes = delegate_output_dtypes(lowered_module) + + def __call__(self, *inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: + device_inputs = [] + for tensor_id, value in zip(self.interpreter.inputs, inputs): + tensor = self.graph.tensors[tensor_id] + array = value.detach().cpu().numpy() + if value.dtype == torch.float32: + array = reference.quantize(tensor, array) + elif value.dtype != torch.int8: + raise TypeError( + f"Delegate input must be float32 or int8, got {value.dtype}" + ) + device_inputs.append( + reference.to_device_layout(tensor, array.astype(np.int8)) + ) + outputs = self.interpreter.run_outputs(device_inputs) + results = [] + for tensor_id, value, dtype in zip( + self.interpreter.outputs, outputs, self.output_dtypes + ): + tensor = self.graph.tensors[tensor_id] + logical = reference.to_logical_layout(tensor, value) + if dtype == torch.float32: + results.append(torch.from_numpy(reference.dequantize(tensor, logical))) + else: + results.append(torch.from_numpy(logical.astype(np.int8))) + return tuple(results) + + +class ProgramEmulator(torch.fx.Interpreter): + """ + Host model of an ExecuTorch program lowered with EspNNPartitioner: + delegates run on the bit-exact ESP-NN reference, every other operator + runs in PyTorch (the device uses the ExecuTorch portable kernels). + """ + + def __init__(self, exported_program): + module = exported_program.module() + super().__init__(module) + self.delegates: dict[str, DelegateEmulator] = {} + self.delegate_calls = 0 + self.cpu_calls = 0 + + def call_function(self, target, args, kwargs): + if "executorch_call_delegate" in str(target): + lowered = args[0] + key = id(lowered) + if key not in self.delegates: + self.delegates[key] = DelegateEmulator(lowered) + self.delegate_calls += 1 + return self.delegates[key](*args[1:]) + if "getitem" not in str(target): + self.cpu_calls += 1 + return super().call_function(target, args, kwargs) + + def __call__(self, *inputs): + return self.run(*inputs) + + +__all__ = ["DelegateEmulator", "ProgramEmulator", "delegate_output_dtypes"] diff --git a/backends/espressif/serialization/__init__.py b/backends/espressif/serialization/__init__.py new file mode 100644 index 00000000000..b3a94f6980b --- /dev/null +++ b/backends/espressif/serialization/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .xts3 import build_graph, parse_graph, serialize_graph, XTS3_VERSION + +__all__ = ["XTS3_VERSION", "build_graph", "parse_graph", "serialize_graph"] diff --git a/backends/espressif/serialization/int8_ops.py b/backends/espressif/serialization/int8_ops.py new file mode 100644 index 00000000000..6dc75184020 --- /dev/null +++ b/backends/espressif/serialization/int8_ops.py @@ -0,0 +1,689 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import math +import struct + +import torch + +from .xts3 import ( + _activation_range, + _fusable_activation, + _normalize_int_tuple, + _per_channel_weight_scales, + _quantize_multiplier, + _require_output_qparams, + _serialize_per_channel_requantization, + _shape_from_node, + _target_kind, + COPY_FLAG_FILL, + COPY_HEADER, + COPY_MAX_RANK, + COPY_SEGMENT, + TensorRegistry, + TRANSPOSE_CONV_PARAMS, + UPSAMPLE_HEADER, + UPSAMPLE_MODE_BILINEAR, + UPSAMPLE_MODE_NEAREST, + UPSAMPLE_WEIGHT_ONE, + XTS3DType, + XTS3OpCode, + XTS3Operation, + XTS3Tensor, +) + + +MOVEMENT_KINDS = { + "reshape", + "flatten", + "unflatten", + "permute", + "transpose", + "squeeze", + "unsqueeze", + "slice", + "select", + "expand", + "split_item", + "identity_move", +} + +SPLIT_KINDS = {"split", "split_with_sizes", "unbind", "chunk"} + + +def is_dequantize(node) -> bool: + return ( + hasattr(node, "op") + and node.op == "call_function" + and "dequantize_per_tensor" in str(node.target) + ) + + +def is_quantize(node) -> bool: + return ( + hasattr(node, "op") + and node.op == "call_function" + and "quantize_per_tensor" in str(node.target) + and "dequantize" not in str(node.target) + ) + + +def node_kind(node) -> str | None: + if not hasattr(node, "op") or node.op != "call_function": + return None + kind = _target_kind(node.target) + if ( + kind == "getitem" + and _target_kind(getattr(node.args[0], "target", None)) in SPLIT_KINDS + ): + return "split_item" + if kind in ("identity", "dropout") and node.args: + source = node.args[0] + while is_dequantize(source) and is_quantize(source.args[0]): + source = source.args[0].args[0] + if node_kind(source) in MOVEMENT_KINDS: + return "identity_move" + return kind + + +def _skip_quantization(node): + while is_dequantize(node) and is_quantize(node.args[0]): + node = node.args[0].args[0] + while node_kind(node) in ("identity", "dropout"): + node = node.args[0] + while is_dequantize(node) and is_quantize(node.args[0]): + node = node.args[0].args[0] + return node + + +def consumers(node) -> list: + result = [] + for user in node.users: + if is_quantize(user): + for dq in user.users: + if is_dequantize(dq): + result.extend(consumers(dq)) + else: + result.append(dq) + elif node_kind(user) in ("identity", "dropout"): + result.extend(consumers(user)) + else: + result.append(user) + return result + + +def is_movement_terminal(node) -> bool: + users = consumers(node) + if not users: + return True + return any(node_kind(user) not in MOVEMENT_KINDS for user in users) + + +def _movement_chain(node): + chain = [] + current = node + while node_kind(current) in MOVEMENT_KINDS: + chain.append(current) + if node_kind(current) == "split_item": + current = _skip_quantization(current.args[0].args[0]) + else: + current = _skip_quantization(current.args[0]) + chain.reverse() + return current, chain + + +def _logical_index_view(shape: tuple[int, ...]) -> torch.Tensor: + count = max(1, math.prod(shape)) + base = torch.arange(count, dtype=torch.int64) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError(f"Rank-4 int8 tensors must have batch 1, got {shape}") + return base.view(shape[2], shape[3], shape[1]).permute(2, 0, 1).unsqueeze(0) + return base.view(shape) + + +def _device_view(view: torch.Tensor) -> torch.Tensor: + if view.dim() == 4: + if view.shape[0] != 1: + raise ValueError( + f"Rank-4 int8 tensors must have batch 1, got {tuple(view.shape)}" + ) + return view[0].permute(1, 2, 0) + return view + + +def _resolved_shape(view: torch.Tensor, shape) -> list[int]: + shape = [int(dim) for dim in shape] + if shape.count(-1) == 1: + known = math.prod(dim for dim in shape if dim != -1) + shape[shape.index(-1)] = view.numel() // max(known, 1) + return shape + + +def _split_bounds(split_node, index: int) -> tuple[int, int, int, bool]: + kind = _target_kind(split_node.target) + source_shape = _shape_from_node(split_node.args[0]) + if kind == "unbind": + dim = int(split_node.args[1]) if len(split_node.args) > 1 else 0 + dim = dim % len(source_shape) + return dim, index, index + 1, True + dim = ( + int(split_node.args[2]) + if len(split_node.args) > 2 + else int(split_node.kwargs.get("dim", 0)) + ) + dim = dim % len(source_shape) + size = source_shape[dim] + if kind == "chunk": + chunks = int(split_node.args[1]) + step = (size + chunks - 1) // chunks + sizes = [min(step, size - start) for start in range(0, size, step)] + elif kind == "split_with_sizes" or isinstance(split_node.args[1], (list, tuple)): + sizes = [int(v) for v in split_node.args[1]] + else: + step = int(split_node.args[1]) + sizes = [min(step, size - start) for start in range(0, size, step)] + start = sum(sizes[:index]) + return dim, start, start + sizes[index], False + + +def _apply_movement(view: torch.Tensor, node): # noqa: C901 + kind = node_kind(node) + args = node.args + if kind in ("reshape", "flatten", "unflatten"): + if kind == "reshape": + shape = _resolved_shape(view, args[1]) + else: + shape = list(_shape_from_node(node)) + try: + return view.view(shape), None + except RuntimeError: + return None, shape + if kind == "permute": + return view.permute([int(d) for d in args[1]]), None + if kind == "transpose": + return view.transpose(int(args[1]), int(args[2])), None + if kind == "squeeze": + if len(args) > 1: + dims = args[1] if isinstance(args[1], (list, tuple)) else [args[1]] + return view.squeeze(tuple(int(d) for d in dims)), None + return view.squeeze(), None + if kind == "unsqueeze": + return view.unsqueeze(int(args[1])), None + if kind == "slice": + dim = int(args[1]) if len(args) > 1 else 0 + start = args[2] if len(args) > 2 else None + end = args[3] if len(args) > 3 else None + step = int(args[4]) if len(args) > 4 else 1 + return torch.ops.aten.slice.Tensor(view, dim, start, end, step), None + if kind == "select": + return view.select(int(args[1]), int(args[2])), None + if kind == "expand": + return view.expand([int(d) for d in args[1]]), None + if kind == "identity_move": + return view, None + if kind == "split_item": + dim, start, end, squeeze = _split_bounds(node.args[0], int(node.args[1])) + sliced = torch.ops.aten.slice.Tensor(view, dim, start, end, 1) + return (sliced.squeeze(dim) if squeeze else sliced), None + raise ValueError(f"Unsupported data movement {node.name}") + + +def _coalesce(dims, in_strides, out_strides): + items = [ + (int(d), int(i), int(o)) + for d, i, o in zip(dims, in_strides, out_strides) + if int(d) != 1 + ] + merged = [] + for dim, in_stride, out_stride in items: + if merged: + p_dim, p_in, p_out = merged[-1] + if p_in == in_stride * dim and p_out == out_stride * dim: + merged[-1] = (p_dim * dim, in_stride, out_stride) + continue + merged.append((dim, in_stride, out_stride)) + if not merged: + merged = [(1, 1, 1)] + return [m[0] for m in merged], [m[1] for m in merged], [m[2] for m in merged] + + +def _contiguous_strides(dims) -> list[int]: + strides = [] + running = 1 + for dim in reversed(dims): + strides.append(running) + running *= int(dim) + return list(reversed(strides)) + + +def copy_segment( + input_index: int, + dims, + in_strides, + in_offset: int, + out_strides, + out_offset: int, + source: XTS3Tensor, + result: XTS3Tensor, +) -> bytes: + dims, in_strides, out_strides = _coalesce(dims, in_strides, out_strides) + if len(dims) > COPY_MAX_RANK: + raise ValueError( + f"Data movement needs {len(dims)} dimensions; the limit is {COPY_MAX_RANK}" + ) + requant = ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ) + multiplier, shift = (0, 0) + if requant: + multiplier, shift = _quantize_multiplier( + source.quant_scale / result.quant_scale + ) + pad = COPY_MAX_RANK - len(dims) + return struct.pack( + COPY_SEGMENT, + input_index, + len(dims), + int(in_offset), + int(out_offset), + *(list(dims) + [1] * pad), + *(list(in_strides) + [0] * pad), + *(list(out_strides) + [0] * pad), + int(requant), + multiplier, + shift, + source.quant_zero_point, + result.quant_zero_point, + ) + + +def copy_operation(inputs, output_id, segments, fill_value=None) -> XTS3Operation: + flags = COPY_FLAG_FILL if fill_value is not None else 0 + header = struct.pack(COPY_HEADER, len(segments), flags, int(fill_value or 0)) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.COPY, + inputs=list(inputs), + outputs=[output_id], + params=header + b"".join(segments), + ) + + +def _gather_operation(source_id, output_id, view, registry): + source = registry.tensors[source_id] + result = registry.tensors[output_id] + device = _device_view(view) + segment = copy_segment( + 0, + list(device.shape), + list(device.stride()), + device.storage_offset(), + _contiguous_strides(device.shape), + 0, + source, + result, + ) + return copy_operation([source_id], output_id, [segment]) + + +def _new_int8_tensor(registry: TensorRegistry, shape, like: XTS3Tensor) -> int: + tensor_id = len(registry.tensors) + registry.tensors.append( + XTS3Tensor( + tensor_id=tensor_id, + shape=tuple(int(d) for d in shape), + dtype=XTS3DType.INT8, + quant_scale=like.quant_scale, + quant_zero_point=like.quant_zero_point, + ) + ) + return tensor_id + + +def lower_movement( + node, registry: TensorRegistry, extra_operations: list +) -> XTS3Operation | None: + if not is_movement_terminal(node): + registry.fused_nodes.add(node) + return None + _require_output_qparams(registry, node) + source_node, chain = _movement_chain(node) + source_id = registry.get_or_create(source_node) + view = _logical_index_view(registry.tensors[source_id].shape) + current_id = source_id + for link in chain: + moved, reshape_to = _apply_movement(view, link) + if moved is None: + temp_id = _new_int8_tensor( + registry, (view.numel(),), registry.tensors[current_id] + ) + temp = registry.tensors[temp_id] + segment = copy_segment( + 0, + list(view.shape), + list(view.stride()), + view.storage_offset(), + _contiguous_strides(view.shape), + 0, + registry.tensors[current_id], + temp, + ) + extra_operations.append(copy_operation([current_id], temp_id, [segment])) + current_id = temp_id + view = torch.arange(view.numel(), dtype=torch.int64).view(reshape_to) + else: + view = moved + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + if tuple(view.shape) != tuple(result.shape): + raise ValueError( + f"Data movement {node.name} traced to {tuple(view.shape)}, expected {result.shape}" + ) + return _gather_operation(current_id, output_id, view, registry) + + +def lower_cat( + node, registry: TensorRegistry, extra_operations: list, stack: bool = False +) -> XTS3Operation: + sources = list(node.args[0]) + dim = int(node.args[1]) if len(node.args) > 1 else int(node.kwargs.get("dim", 0)) + _require_output_qparams(registry, node) + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + rank = len(result.shape) + dim = dim % rank + out_view = _logical_index_view(result.shape) + input_ids = [registry.get_or_create(source) for source in sources] + operations = [] + offset = 0 + pending_inputs = [] + pending_segments = [] + for input_id in input_ids: + source = registry.tensors[input_id] + if stack: + if len(source.shape) != rank - 1: + raise ValueError(f"stack {node.name} mixes ranks") + part = out_view.select(dim, offset) + offset += 1 + else: + if len(source.shape) != rank: + raise ValueError(f"cat {node.name} mixes ranks") + size = source.shape[dim] + part = torch.ops.aten.slice.Tensor(out_view, dim, offset, offset + size, 1) + offset += size + if source.shape and math.prod(source.shape) == 0: + continue + in_device = _device_view(_logical_index_view(source.shape)) + out_device = _device_view(part) + dims = list(in_device.shape) + if len(pending_inputs) == 6: + operations.append( + copy_operation(pending_inputs, output_id, pending_segments) + ) + pending_inputs, pending_segments = [], [] + pending_segments.append( + copy_segment( + len(pending_inputs), + dims, + list(in_device.stride()), + in_device.storage_offset(), + list(out_device.stride()), + out_device.storage_offset(), + source, + result, + ) + ) + pending_inputs.append(input_id) + operations.append(copy_operation(pending_inputs, output_id, pending_segments)) + extra_operations.extend(operations[:-1]) + return operations[-1] + + +def lower_pad(node, registry: TensorRegistry) -> XTS3Operation: + pads = [int(v) for v in node.args[1]] + value = ( + float(node.args[2]) + if len(node.args) > 2 + else float(node.kwargs.get("value", 0.0)) + ) + if any(p < 0 for p in pads): + raise ValueError(f"Negative padding (cropping) is not supported: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + rank = len(result.shape) + out_view = _logical_index_view(result.shape) + region = out_view + for i in range(len(pads) // 2): + dim = rank - 1 - i + before = pads[2 * i] + region = torch.ops.aten.slice.Tensor( + region, dim, before, before + source.shape[dim], 1 + ) + in_device = _device_view(_logical_index_view(source.shape)) + out_device = _device_view(region) + fill = round(value / result.quant_scale) + result.quant_zero_point + fill = max(-128, min(127, fill)) + segment = copy_segment( + 0, + list(in_device.shape), + list(in_device.stride()), + 0, + list(out_device.stride()), + out_device.storage_offset(), + source, + result, + ) + return copy_operation([input_id], output_id, [segment], fill_value=fill) + + +def broadcast_operand( + arg, target_shape, registry: TensorRegistry, extra_operations: list +) -> int: + tensor_id = registry.get_or_create(arg) + tensor = registry.tensors[tensor_id] + if tuple(tensor.shape) == tuple(target_shape): + return tensor_id + if tensor.is_constant: + return registry.broadcast_constant(tensor_id, tuple(target_shape)) + shape = (1,) * (len(target_shape) - len(tensor.shape)) + tuple(tensor.shape) + view = _logical_index_view(tensor.shape).reshape(shape).expand(*target_shape) + temp_id = _new_int8_tensor(registry, target_shape, tensor) + extra_operations.append(_gather_operation(tensor_id, temp_id, view, registry)) + return temp_id + + +def upsample_tables(node, in_size: int, out_size: int, axis: int, bilinear: bool): + args = node.args + kind = _target_kind(node.target) + scales = None + align_corners = False + if bilinear: + align_corners = ( + bool(args[2]) + if len(args) > 2 + else bool(node.kwargs.get("align_corners", False)) + ) + if kind.endswith("_vec"): + scales = args[3] if len(args) > 3 else node.kwargs.get("scale_factors") + else: + factor = args[3 + axis] if len(args) > 3 + axis else None + scales = None if factor is None else [factor, factor] + else: + if kind.endswith("_vec"): + scales = args[2] if len(args) > 2 else node.kwargs.get("scale_factors") + else: + factor = args[2 + axis] if len(args) > 2 + axis else None + scales = None if factor is None else [factor, factor] + scale_factor = None + if scales is not None and len(scales) == 2 and scales[axis] is not None: + scale_factor = float(scales[axis]) + table = [] + if not bilinear: + ratio = (1.0 / scale_factor) if scale_factor else in_size / out_size + for dst in range(out_size): + src = min(int(math.floor(dst * ratio)), in_size - 1) + table.extend([src, src, 0]) + return table + if align_corners: + ratio = (in_size - 1) / (out_size - 1) if out_size > 1 else 0.0 + else: + ratio = (1.0 / scale_factor) if scale_factor else in_size / out_size + for dst in range(out_size): + if align_corners: + src = dst * ratio + else: + src = max((dst + 0.5) * ratio - 0.5, 0.0) + x0 = min(int(math.floor(src)), in_size - 1) + x1 = x0 + 1 if x0 < in_size - 1 else x0 + weight = int(round((src - x0) * UPSAMPLE_WEIGHT_ONE)) if x1 != x0 else 0 + table.extend([x0, x1, weight]) + return table + + +def lower_upsample(node, registry: TensorRegistry) -> XTS3Operation: + kind = _target_kind(node.target) + bilinear = kind.startswith("upsample_bilinear2d") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if len(source.shape) != 4 or source.shape[0] != 1: + raise ValueError(f"Upsampling needs a batch-1 NCHW tensor: {node.name}") + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError(f"Upsampling {node.name} must keep its input quantization") + in_h, in_w = source.shape[2], source.shape[3] + out_h, out_w = result.shape[2], result.shape[3] + y_table = upsample_tables(node, in_h, out_h, 0, bilinear) + x_table = upsample_tables(node, in_w, out_w, 1, bilinear) + params = ( + struct.pack( + UPSAMPLE_HEADER, + UPSAMPLE_MODE_BILINEAR if bilinear else UPSAMPLE_MODE_NEAREST, + in_h, + in_w, + out_h, + out_w, + ) + + struct.pack(f"<{len(y_table)}i", *y_table) + + struct.pack(f"<{len(x_table)}i", *x_table) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.UPSAMPLE, + inputs=[input_id], + outputs=[output_id], + params=params, + ) + + +def transposed_conv_args(node) -> dict: + kind = _target_kind(node.target) + args = node.args + if kind == "convolution": + return { + "input": args[0], + "weight": args[1], + "bias": args[2], + "stride": _normalize_int_tuple(args[3], 2, "stride"), + "padding": _normalize_int_tuple(args[4], 2, "padding"), + "dilation": _normalize_int_tuple(args[5], 2, "dilation"), + "output_padding": _normalize_int_tuple(args[7], 2, "output_padding"), + "groups": int(args[8]), + } + return { + "input": args[0], + "weight": args[1], + "bias": args[2] if len(args) > 2 else None, + "stride": _normalize_int_tuple(args[3] if len(args) > 3 else 1, 2, "stride"), + "padding": _normalize_int_tuple(args[4] if len(args) > 4 else 0, 2, "padding"), + "output_padding": _normalize_int_tuple( + args[5] if len(args) > 5 else 0, 2, "output_padding" + ), + "groups": int(args[6]) if len(args) > 6 else 1, + "dilation": _normalize_int_tuple( + args[7] if len(args) > 7 else 1, 2, "dilation" + ), + } + + +def lower_transposed_conv(node, registry: TensorRegistry) -> XTS3Operation: + conv = transposed_conv_args(node) + if conv["groups"] != 1: + raise ValueError( + f"Grouped transposed convolution is not supported: {node.name}" + ) + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_id = registry.get_or_create(conv["input"]) + weight_scales = _per_channel_weight_scales(registry, conv["weight"]) + weight_id = registry.get_or_create(conv["weight"], "TRANSPOSED") + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + conv["bias"], input_tensor.quant_scale, weight_scales + ) + output_id = registry.get_or_create(output_node) + output_tensor = registry.tensors[output_id] + if activation is not None: + registry.fused_nodes.add(activation) + weight_shape = _shape_from_node(conv["weight"]) + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack( + TRANSPOSE_CONV_PARAMS, + weight_shape[2], + weight_shape[3], + *conv["stride"], + *conv["padding"], + *conv["output_padding"], + *conv["dilation"], + conv["groups"], + act_min, + act_max, + ) + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.TRANSPOSE_CONV2D, + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + +__all__ = [ + "MOVEMENT_KINDS", + "SPLIT_KINDS", + "broadcast_operand", + "copy_operation", + "copy_segment", + "is_dequantize", + "is_movement_terminal", + "is_quantize", + "lower_cat", + "lower_movement", + "lower_pad", + "lower_transposed_conv", + "lower_upsample", + "node_kind", + "transposed_conv_args", + "upsample_tables", +] diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py new file mode 100644 index 00000000000..b3f7a07d41e --- /dev/null +++ b/backends/espressif/serialization/xts3.py @@ -0,0 +1,2048 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from enum import IntEnum +from math import frexp, prod + +import torch + + +MAGIC = b"XTS3" +XTS3_VERSION = 6 +XTS3_VERSIONS = (5, 6) +IO_TABLE_HEADER = " int: + if dtype == XTS3DType.FLOAT32: + return 4 + if dtype == XTS3DType.INT8: + return 1 + if dtype == XTS3DType.UINT8: + return 1 + if dtype == XTS3DType.INT32: + return 4 + raise ValueError(f"Unsupported ESP32-S3 dtype: {dtype}") + + +TENSOR_FLAG_CONSTANT = 1 << 0 +TENSOR_FLAG_QUANTIZED = 1 << 1 +TENSOR_FLAG_GRAPH_INPUT = 1 << 2 +TENSOR_FLAG_GRAPH_OUTPUT = 1 << 3 + + +@dataclass +class XTS3Tensor: + tensor_id: int + shape: tuple[int, ...] + dtype: XTS3DType + data_offset: int = 0 + data_size: int = 0 + flags: int = 0 + quant_scale: float = 0.0 + quant_zero_point: int = 0 + + @property + def is_constant(self) -> bool: + return bool(self.flags & TENSOR_FLAG_CONSTANT) + + +@dataclass +class XTS3Operation: + op_id: int + opcode: XTS3OpCode + inputs: list[int] = field(default_factory=list) + outputs: list[int] = field(default_factory=list) + params: bytes = b"" + + +@dataclass +class XTS3Graph: + tensors: list[XTS3Tensor] = field(default_factory=list) + operations: list[XTS3Operation] = field(default_factory=list) + constant_data: bytes = b"" + version: int = XTS3_VERSION + tensor_names: dict[int, str] = field(default_factory=dict) + inputs: list[int] = field(default_factory=list) + outputs: list[int] = field(default_factory=list) + + +class ExportedTensorResolver: + def __init__(self, exported_program): + self.exported_program = exported_program + self.graph_signature = getattr( + exported_program, + "graph_signature", + None, + ) + self.state_dict = ( + getattr( + exported_program, + "state_dict", + {}, + ) + or {} + ) + + def _lookup_mapping(self, mapping, name): + if mapping is None: + return None + + if name in mapping: + return mapping[name] + + for key, value in mapping.items(): + if str(key) == str(name): + return value + + return None + + def resolve(self, node): # noqa: C901 + if node is not None and node.op == "call_function": + target = str(node.target) + if "dequantize_per_channel" in target: + quantized = self.resolve(node.args[0]) + scales = self.resolve(node.args[1]) + zero_points = self.resolve(node.args[2]) + axis = int(node.args[3]) + if quantized is None or scales is None or zero_points is None: + return None + view = [1] * quantized.dim() + view[axis] = -1 + scale_values = scales.to(torch.float32).reshape(view) + zero_values = zero_points.to(torch.float32).reshape(view) + return (quantized.to(torch.float32) - zero_values) * scale_values + if node.op != "placeholder": + return None + + name = node.name + target = node.target + + parameter_mapping = getattr( + self.graph_signature, + "inputs_to_parameters", + {}, + ) + + buffer_mapping = getattr( + self.graph_signature, + "inputs_to_buffers", + {}, + ) + + state_name = self._lookup_mapping( + parameter_mapping, + name, + ) + + if state_name is None: + state_name = self._lookup_mapping( + parameter_mapping, + target, + ) + + if state_name is None: + state_name = self._lookup_mapping( + buffer_mapping, + name, + ) + + if state_name is None: + state_name = self._lookup_mapping( + buffer_mapping, + target, + ) + + if state_name is not None and state_name in self.state_dict: + return self.state_dict[state_name] + + constants = getattr(self.exported_program, "constants", None) or {} + constant_mapping = getattr( + self.graph_signature, + "inputs_to_lifted_tensor_constants", + {}, + ) + constant_name = self._lookup_mapping(constant_mapping, name) + if constant_name is None: + constant_name = self._lookup_mapping(constant_mapping, target) + if constant_name is not None and constant_name in constants: + return constants[constant_name] + + for candidate in (target, name): + if candidate in self.state_dict: + return self.state_dict[candidate] + if candidate in constants: + return constants[candidate] + + return None + + +def _shape_from_node(node) -> tuple[int, ...]: + value = node.meta.get("val") + + if value is None: + return () + + shape = getattr( + value, + "shape", + None, + ) + + if shape is None: + return () + + return tuple(int(dim) for dim in shape) + + +def _dtype_from_node(node) -> torch.dtype: + value = node.meta.get("val") + + dtype = getattr( + value, + "dtype", + None, + ) + + if dtype is None: + raise ValueError(f"Node {node.name} has no dtype metadata") + + return dtype + + +def _normalize_int_tuple( + value, + length: int, + name: str, +) -> tuple[int, ...]: + + if isinstance(value, int): + return (int(value),) * length + + result = tuple(int(x) for x in value) + + if len(result) != length: + raise ValueError(f"Expected {length} values for " f"{name}, got {result}") + + return result + + +def _target_text(target) -> str: + return str(target) + + +LUT_KINDS = ( + "hardswish", + "hardsigmoid", + "sigmoid", + "silu", + "tanh", + "gelu", + "elu", + "leaky_relu", +) + + +def _target_kind(target) -> str | None: # noqa: C901 + text = _target_text(target) + + if "aten.convolution.default" in text: + return "convolution" + + if "aten.conv_transpose2d" in text: + return "conv_transpose2d" + + if "aten.upsample_nearest2d.vec" in text: + return "upsample_nearest2d_vec" + + if "aten.upsample_nearest2d.default" in text: + return "upsample_nearest2d" + + if "aten.upsample_bilinear2d.vec" in text: + return "upsample_bilinear2d_vec" + + if "aten.upsample_bilinear2d.default" in text: + return "upsample_bilinear2d" + + if "aten.constant_pad_nd.default" in text or "aten.pad.default" in text: + return "pad" + + for name, kind in ( + ("permute", "permute"), + ("transpose", "transpose"), + ("unsqueeze", "unsqueeze"), + ("squeeze", "squeeze"), + ("expand", "expand"), + ("unflatten", "unflatten"), + ("split_with_sizes", "split_with_sizes"), + ("unbind", "unbind"), + ("chunk", "chunk"), + ): + if f"aten.{name}." in text or f"aten.{name}_copy." in text: + return kind + + if "aten.split." in text or "aten.split_copy." in text: + return "split" + + if "aten.slice.Tensor" in text or "aten.slice_copy.Tensor" in text: + return "slice" + + if "aten.select.int" in text or "aten.select_copy.int" in text: + return "select" + + if "aten.conv2d.default" in text: + return "conv2d" + + if "aten.hardtanh_.default" in text: + return "hardtanh_" + + if "aten.hardtanh.default" in text: + return "hardtanh" + + if "aten.relu_.default" in text or "aten.relu.default" in text: + return "relu" + + if "aten.relu6.default" in text: + return "relu6" + + if "aten.add.Tensor" in text or "aten.add_.Tensor" in text: + return "add" + + if "aten.mul.Tensor" in text or "aten.mul_.Tensor" in text: + return "mul" + + if "aten._softmax.default" in text or "aten.softmax.int" in text: + return "softmax" + + for name in LUT_KINDS: + if f"aten.{name}.default" in text or f"aten.{name}_.default" in text: + return name + + if "aten.adaptive_avg_pool2d.default" in text: + return "adaptive_avg_pool2d" + + if "aten.max_pool2d_with_indices.default" in text: + return "max_pool2d_with_indices" + + if "aten.max_pool2d.default" in text: + return "max_pool2d" + + if "aten.avg_pool2d.default" in text: + return "avg_pool2d" + + if "aten.cat.default" in text: + return "cat" + + if "aten.stack.default" in text: + return "stack" + + if any( + marker in text + for marker in ( + "aten.clone.default", + "_clone_dim_order.default", + "aten.alias.default", + "aten.alias_copy.default", + "aten.detach.default", + "aten.detach_copy.default", + ) + ): + return "identity" + + if "aten.dropout.default" in text or "aten.dropout_.default" in text: + return "dropout" + + if text == "" or text.endswith("operator.getitem"): + return "getitem" + + if any( + marker in text + for marker in ( + "aten.view.default", + "aten.view_copy.default", + "aten.reshape.default", + "aten._unsafe_view.default", + ) + ): + return "reshape" + + if "aten.mean.dim" in text: + return "mean_dim" + + if "aten.flatten.using_ints" in text: + return "flatten" + + if "aten.linear.default" in text: + return "linear" + + return None + + +def _int8_device_bytes(values: torch.Tensor, shape: tuple[int, ...]) -> bytes: + values = values.to(torch.int8).reshape(shape) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError(f"Rank-4 int8 constants must have batch 1, got {shape}") + values = values[0].permute(1, 2, 0) + return values.contiguous().numpy().tobytes() + + +def _int8_from_device(values: torch.Tensor, shape: tuple[int, ...]) -> torch.Tensor: + if len(shape) == 4: + return ( + values.reshape(shape[2], shape[3], shape[1]).permute(2, 0, 1).unsqueeze(0) + ) + return values.reshape(shape) + + +class TensorRegistry: + def __init__(self, resolver: ExportedTensorResolver, graph_module): + self.resolver = resolver + self.tensors: list[XTS3Tensor] = [] + self.node_to_id: dict[object, int] = {} + self.constant_data = bytearray() + self.fused_nodes: set = set() + self.extra_operations: list = [] + self.node_to_qparams: dict[object, tuple[float, int]] = {} + for node in graph_module.graph.nodes: + if node.op != "call_function" or "quantize_per_tensor" not in str( + node.target + ): + continue + if len(node.args) < 3 or not hasattr(node.args[0], "meta"): + continue + self.node_to_qparams[node.args[0]] = ( + float(node.args[1]), + int(node.args[2]), + ) + + def _align_constant_data(self, alignment: int = 16) -> None: + self.constant_data.extend(b"\x00" * ((-len(self.constant_data)) % alignment)) + + def get_or_create(self, node, filter_layout: str | None = None) -> int: + return self._get_or_create_int8(node, filter_layout) + + def _append_constant(self, tensor: XTS3Tensor, raw: bytes) -> int: + self._align_constant_data() + tensor.data_offset = len(self.constant_data) + tensor.data_size = len(raw) + tensor.flags |= TENSOR_FLAG_CONSTANT + self.constant_data.extend(raw) + self.tensors.append(tensor) + return tensor.tensor_id + + def _new_int8_activation(self, node, scale: float, zero_point: int) -> int: + if node in self.node_to_id: + tensor_id = self.node_to_id[node] + tensor = self.tensors[tensor_id] + if tensor.quant_scale != scale or tensor.quant_zero_point != zero_point: + raise ValueError( + f"Tensor {node.name} has inconsistent activation qparams" + ) + return tensor_id + shape = _shape_from_node(node) + if not shape: + raise ValueError(f"Quantized activation {node.name} has no shape metadata") + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + quant_scale=float(scale), + quant_zero_point=int(zero_point), + ) + self.tensors.append(tensor) + self.node_to_id[node] = tensor_id + return tensor_id + + def _constant_int8(self, node) -> int: + quantized_node = node.args[0] + if quantized_node in self.node_to_id: + return self.node_to_id[quantized_node] + quantized = self.resolver.resolve(quantized_node) + if quantized is None or quantized.dtype != torch.int8: + raise ValueError( + f"Expected quantize_per_tensor or an int8 constant before {node.name}" + ) + shape = tuple(int(dim) for dim in quantized.shape) + if len(shape) > 4: + raise ValueError( + f"Constant {node.name} has rank {len(shape)}; the limit is 4" + ) + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + quant_scale=float(node.args[1]), + quant_zero_point=int(node.args[2]), + ) + self._append_constant( + tensor, _int8_device_bytes(quantized.detach().cpu(), shape) + ) + self.node_to_id[quantized_node] = tensor_id + return tensor_id + + def broadcast_constant(self, tensor_id: int, shape: tuple[int, ...]) -> int: + source = self.tensors[tensor_id] + values = torch.frombuffer( + bytearray( + self.constant_data[ + source.data_offset : source.data_offset + source.data_size + ] + ), + dtype=torch.int8, + ) + logical = _int8_from_device(values, source.shape) + expanded = ( + logical.reshape( + (1,) * (len(shape) - len(source.shape)) + tuple(source.shape) + ) + .expand(shape) + .contiguous() + ) + result_id = len(self.tensors) + result = XTS3Tensor( + tensor_id=result_id, + shape=tuple(shape), + dtype=XTS3DType.INT8, + quant_scale=source.quant_scale, + quant_zero_point=source.quant_zero_point, + ) + self._append_constant(result, _int8_device_bytes(expanded, tuple(shape))) + return result_id + + def _get_or_create_int8( # noqa: C901 + self, node, filter_layout: str | None = None + ) -> int: + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized_node = node.args[0] + if not ( + getattr(quantized_node, "op", None) == "call_function" + and "quantize_per_tensor" in str(quantized_node.target) + ): + if ( + quantized_node.op == "placeholder" + and self.resolver.resolve(quantized_node) is None + ): + return self._new_int8_activation( + quantized_node, float(node.args[1]), int(node.args[2]) + ) + return self._constant_int8(node) + return self._new_int8_activation( + quantized_node.args[0], + float(node.args[1]), + int(node.args[2]), + ) + + if node.op == "call_function" and "dequantize_per_channel" in str(node.target): + quantized_node = node.args[0] + if quantized_node in self.node_to_id: + return self.node_to_id[quantized_node] + quantized = self.resolver.resolve(quantized_node) + if quantized is None or quantized.dtype != torch.int8: + raise ValueError( + f"Quantized weight {node.name} is not backed by int8 data" + ) + expected_axis = 1 if filter_layout == "TRANSPOSED" else 0 + if int(node.args[3]) != expected_axis: + raise ValueError( + f"Only output-channel weight quantization is supported: {node.name}" + ) + if not torch.all(self.resolver.resolve(node.args[2]) == 0): + raise ValueError( + f"Only symmetric int8 weights are supported: {node.name}" + ) + shape = tuple(int(dim) for dim in quantized.shape) + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + ) + raw = quantized.detach().cpu().contiguous() + if len(shape) == 4 and filter_layout == "OHWI": + raw = raw.permute(0, 2, 3, 1).contiguous() + elif len(shape) == 4 and filter_layout == "TRANSPOSED": + raw = raw.permute(1, 2, 3, 0).contiguous() + tensor.shape = (shape[1], shape[2], shape[3], shape[0]) + elif len(shape) == 4 and filter_layout == "HWOC": + raw = ( + raw.permute(2, 3, 0, 1) + .reshape(shape[2], shape[3], shape[0]) + .contiguous() + ) + elif len(shape) == 4: + raise ValueError( + f"A convolution filter layout is required for {node.name}" + ) + raw_bytes = raw.numpy().tobytes() + self._append_constant(tensor, raw_bytes) + self.node_to_id[quantized_node] = tensor_id + return tensor_id + + if node in self.node_to_id: + return self.node_to_id[node] + + quantization = self.node_to_qparams.get(node) + if quantization is None: + raise ValueError( + f"Activation {node.name} has no PT2E quantization parameters" + ) + return self._new_int8_activation(node, *quantization) + + def add_int32_bias( + self, node, input_scale: float, weight_scales: torch.Tensor + ) -> int: + scales = weight_scales.detach().cpu().to(torch.float64).reshape(-1) + if node is None: + bias = torch.zeros(scales.numel(), dtype=torch.float64) + else: + bias = self.resolver.resolve(node) + if bias is None: + raise ValueError(f"Unable to resolve bias tensor {node.name}") + bias = bias.detach().cpu().to(torch.float64).reshape(-1) + if bias.numel() != scales.numel(): + raise ValueError("Bias and per-channel weight scale counts differ") + bias_q = torch.round(bias / (float(input_scale) * scales)) + if torch.any(bias_q < -(1 << 31)) or torch.any(bias_q > (1 << 31) - 1): + raise ValueError("Quantized bias overflows int32") + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=(int(bias_q.numel()),), + dtype=XTS3DType.INT32, + ) + raw_bytes = bias_q.to(torch.int32).contiguous().numpy().tobytes() + self._append_constant(tensor, raw_bytes) + return tensor_id + + +def _get_conv_args(node): + args = node.args + + if len(args) < 2: + raise ValueError( + f"Convolution node {node.name} " f"has fewer than two arguments" + ) + + kind = _target_kind(node.target) + + if kind == "convolution": + if len(args) < 9: + raise ValueError( + f"Edge convolution node " + f"{node.name} has unexpected " + f"argument count: {len(args)}" + ) + + transposed = bool(args[6]) + + output_padding = _normalize_int_tuple( + args[7], + 2, + "output_padding", + ) + + if transposed: + raise ValueError( + f"Transposed convolution must use the transposed lowering: {node.name}" + ) + + if any(output_padding): + raise ValueError( + f"Non-zero output_padding is " f"not supported: {node.name}" + ) + + return { + "input": args[0], + "weight": args[1], + "bias": args[2], + "stride": _normalize_int_tuple( + args[3], + 2, + "stride", + ), + "padding": _normalize_int_tuple( + args[4], + 2, + "padding", + ), + "dilation": _normalize_int_tuple( + args[5], + 2, + "dilation", + ), + "groups": int(args[8]), + } + + return { + "input": args[0], + "weight": args[1], + "bias": (args[2] if len(args) > 2 else None), + "stride": _normalize_int_tuple( + args[3] if len(args) > 3 else (1, 1), + 2, + "stride", + ), + "padding": _normalize_int_tuple( + args[4] if len(args) > 4 else (0, 0), + 2, + "padding", + ), + "dilation": _normalize_int_tuple( + args[5] if len(args) > 5 else (1, 1), + 2, + "dilation", + ), + "groups": (int(args[6]) if len(args) > 6 else 1), + } + + +def _is_depthwise_conv( + node, + conv_args, +) -> bool: + + groups = conv_args["groups"] + + weight_shape = _shape_from_node(conv_args["weight"]) + + input_shape = _shape_from_node(conv_args["input"]) + + if len(weight_shape) != 4 or len(input_shape) < 2: + return False + + return groups > 1 and groups == input_shape[1] and weight_shape[1] == 1 + + +def _is_relu6(node) -> bool: + if len(node.args) < 3: + return False + + try: + minimum = float(node.args[1]) + maximum = float(node.args[2]) + except ( + TypeError, + ValueError, + ): + return False + + return minimum == 0.0 and maximum == 6.0 + + +def _quantize_multiplier(real_multiplier: float) -> tuple[int, int]: + if not real_multiplier > 0.0: + raise ValueError( + f"Quantized multiplier must be positive, got {real_multiplier}" + ) + significand, exponent = frexp(real_multiplier) + multiplier = round(significand * (1 << 31)) + if multiplier == (1 << 31): + multiplier >>= 1 + exponent += 1 + if multiplier > 0x7FFFFFFF: + multiplier = 0x7FFFFFFF + if exponent < -31 or exponent > 30: + raise ValueError( + f"Requantization multiplier {real_multiplier} is outside the ESP-NN shift range" + ) + return int(multiplier), int(exponent) + + +def _quantize_multiplier_smaller_than_one(real_multiplier: float) -> tuple[int, int]: + if not 0.0 < real_multiplier < 1.0: + raise ValueError(f"Expected a multiplier in (0, 1), got {real_multiplier}") + multiplier, shift = _quantize_multiplier(real_multiplier) + if shift > 0: + raise ValueError(f"Unexpected positive shift for {real_multiplier}") + return multiplier, shift + + +def _per_channel_weight_scales(registry: TensorRegistry, weight_node) -> torch.Tensor: + if weight_node.op != "call_function" or "dequantize_per_channel" not in str( + weight_node.target + ): + raise ValueError( + f"Expected per-channel quantized weights at {weight_node.name}" + ) + scales = registry.resolver.resolve(weight_node.args[1]) + zero_points = registry.resolver.resolve(weight_node.args[2]) + if scales is None or zero_points is None: + raise ValueError( + f"Unable to resolve per-channel weight qparams for {weight_node.name}" + ) + scales = scales.detach().cpu().to(torch.float64).reshape(-1) + if not torch.all(torch.isfinite(scales)) or torch.any(scales <= 0): + raise ValueError(f"Invalid per-channel weight scales for {weight_node.name}") + if not torch.all(zero_points == 0): + raise ValueError( + f"Only symmetric per-channel weights are supported for {weight_node.name}" + ) + return scales + + +def _serialize_per_channel_requantization( + input_scale: float, + weight_scales: torch.Tensor, + output_scale: float, +) -> bytes: + multipliers = [] + shifts = [] + for weight_scale in weight_scales.tolist(): + multiplier, shift = _quantize_multiplier( + float(input_scale) * float(weight_scale) / float(output_scale) + ) + multipliers.append(multiplier) + shifts.append(shift) + return struct.pack(f"<{len(multipliers)}i", *multipliers) + struct.pack( + f"<{len(shifts)}i", *shifts + ) + + +def _round_half_away(value: float) -> int: + return int(value + 0.5) if value >= 0 else -int(-value + 0.5) + + +def _activation_range( + kind: str | None, scale: float, zero_point: int +) -> tuple[int, int]: + if kind is None: + return -128, 127 + low = max(-128, zero_point) + if kind == "relu": + return low, 127 + if kind == "relu6": + return low, min(127, zero_point + _round_half_away(6.0 / scale)) + raise ValueError(f"Unsupported fused activation {kind}") + + +def _int8_activation_kind(node) -> str | None: + kind = _target_kind(node.target) if node.op == "call_function" else None + if kind == "relu": + return "relu" + if kind == "relu6": + return "relu6" + if kind in ("hardtanh", "hardtanh_") and _is_relu6(node): + return "relu6" + return None + + +def _fusable_activation(node, registry: TensorRegistry): + users = list(node.users) + if len(users) != 1: + return None, None + activation = users[0] + kind = _int8_activation_kind(activation) + if kind is None or activation.args[0] is not node: + return None, None + if activation not in registry.node_to_qparams: + return None, None + return activation, kind + + +def _require_output_qparams(registry: TensorRegistry, node) -> None: + if node not in registry.node_to_qparams: + raise ValueError( + f"Int8 operation {node.name} has no output quantization; " + "quantize the model with EspNNQuantizer before lowering" + ) + + +def lower_int8_node( # noqa: C901 + node, registry: TensorRegistry +) -> XTS3Operation | None: + if node.op != "call_function" or node in registry.fused_nodes: + return None + kind = _target_kind(node.target) + + if kind == "conv_transpose2d" or ( + kind == "convolution" and len(node.args) > 6 and bool(node.args[6]) + ): + from .int8_ops import lower_transposed_conv + + return lower_transposed_conv(node, registry) + + if kind in ( + "upsample_nearest2d", + "upsample_nearest2d_vec", + "upsample_bilinear2d", + "upsample_bilinear2d_vec", + ): + from .int8_ops import lower_upsample + + return lower_upsample(node, registry) + + if kind == "pad": + from .int8_ops import lower_pad + + return lower_pad(node, registry) + + if kind in ("split", "split_with_sizes", "unbind", "chunk"): + registry.fused_nodes.add(node) + return None + + if kind in ( + "permute", + "transpose", + "squeeze", + "unsqueeze", + "slice", + "select", + "expand", + "unflatten", + ) or ( + kind == "getitem" + and _target_kind(getattr(node.args[0], "target", None)) + in ("split", "split_with_sizes", "unbind", "chunk") + ): + from .int8_ops import lower_movement + + return lower_movement(node, registry, registry.extra_operations) + + if kind in ("conv2d", "convolution"): + conv = _get_conv_args(node) + if ( + max(conv["dilation"]) > 255 + or max(conv["padding"]) > 255 + or max(conv["stride"]) > 255 + ): + raise ValueError( + f"Convolution geometry exceeds the 8-bit parameter fields: {node.name}" + ) + depthwise = _is_depthwise_conv(node, conv) + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_id = registry.get_or_create(conv["input"]) + weight_scales = _per_channel_weight_scales(registry, conv["weight"]) + weight_id = registry.get_or_create( + conv["weight"], "HWOC" if depthwise else "OHWI" + ) + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + conv["bias"], input_tensor.quant_scale, weight_scales + ) + output_id = registry.get_or_create(output_node) + output_tensor = registry.tensors[output_id] + if activation is not None: + registry.fused_nodes.add(activation) + weight_shape = _shape_from_node(conv["weight"]) + if len(weight_shape) != 4: + raise ValueError(f"Convolution weight must be rank 4: {node.name}") + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack( + INT8_CONV_BASE_PARAMS, + weight_shape[2], + weight_shape[3], + conv["stride"][0], + conv["stride"][1], + conv["padding"][0], + conv["padding"][1], + conv["dilation"][0], + conv["dilation"][1], + conv["groups"], + act_min, + act_max, + ) + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + return XTS3Operation( + op_id=-1, + opcode=(XTS3OpCode.DEPTHWISE_CONV2D if depthwise else XTS3OpCode.CONV2D), + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + if kind in ("hardtanh_", "hardtanh", "relu", "relu6"): + activation_kind = _int8_activation_kind(node) + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if activation_kind is None or ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + return _lut_operation(node, input_id, output_id, registry) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.RELU6, + inputs=[input_id], + outputs=[output_id], + params=struct.pack(INT8_PAIR_PARAMS, act_min, act_max), + ) + + if kind in LUT_KINDS: + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + return _lut_operation(node, input_id, output_id, registry) + + if kind in ("add", "mul"): + scalar_operation = _lower_scalar_binary(node, kind, registry) + if scalar_operation is not None: + return scalar_operation + + if kind == "mul": + return _lower_int8_mul(node, registry) + + if kind == "softmax": + return _lower_int8_softmax(node, registry) + + if kind == "add": + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + from .int8_ops import broadcast_operand + + output_id = registry.get_or_create(output_node) + result = registry.tensors[output_id] + lhs_id = broadcast_operand( + node.args[0], result.shape, registry, registry.extra_operations + ) + rhs_id = broadcast_operand( + node.args[1], result.shape, registry, registry.extra_operations + ) + if activation is not None: + registry.fused_nodes.add(activation) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + if lhs.shape != rhs.shape or lhs.shape != result.shape: + raise ValueError(f"Broadcasting add could not be resolved: {node.name}") + twice_max_input_scale = 2.0 * max(lhs.quant_scale, rhs.quant_scale) + lhs_mult, lhs_shift = _quantize_multiplier_smaller_than_one( + lhs.quant_scale / twice_max_input_scale + ) + rhs_mult, rhs_shift = _quantize_multiplier_smaller_than_one( + rhs.quant_scale / twice_max_input_scale + ) + out_mult, out_shift = _quantize_multiplier_smaller_than_one( + twice_max_input_scale / ((1 << ADD_LEFT_SHIFT) * result.quant_scale) + ) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.ADD, + inputs=[lhs_id, rhs_id], + outputs=[output_id], + params=struct.pack( + INT8_ADD_PARAMS, + lhs_mult, + lhs_shift, + rhs_mult, + rhs_shift, + ADD_LEFT_SHIFT, + out_mult, + out_shift, + act_min, + act_max, + ), + ) + + if kind in ("adaptive_avg_pool2d", "mean_dim"): + if kind == "adaptive_avg_pool2d": + output_size = ( + tuple(int(v) for v in node.args[1]) if len(node.args) > 1 else () + ) + if output_size != (1, 1): + return _lower_adaptive_avg_pool(node, output_size, registry) + else: + if ( + len(node.args) < 3 + or tuple(int(d) for d in node.args[1]) not in ((2, 3), (-2, -1)) + or not bool(node.args[2]) + ): + raise ValueError(f"Only spatial keepdim mean is supported: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if len(source.shape) != 4: + raise ValueError(f"Average pooling input must be rank 4: {node.name}") + elements = source.shape[2] * source.shape[3] + mult, shift = _quantize_multiplier( + source.quant_scale / (result.quant_scale * elements) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.AVGPOOL2D, + inputs=[input_id], + outputs=[output_id], + params=struct.pack(INT8_PAIR_PARAMS, mult, shift), + ) + + if kind in ("max_pool2d", "avg_pool2d"): + return _lower_int8_pool(node, kind, registry) + + if kind == "max_pool2d_with_indices": + users = list(node.users) + if any( + _target_kind(user.target) != "getitem" or user.args[1] != 0 + for user in users + ): + raise ValueError( + f"max_pool2d indices are used by {node.name}; only values are supported" + ) + if len(users) != 1: + raise ValueError( + f"max_pool2d_with_indices {node.name} must have one getitem user" + ) + operation = _lower_int8_pool(users[0], "max_pool2d", registry, source_node=node) + registry.fused_nodes.add(users[0]) + return operation + + if kind in ("identity", "dropout"): + from .int8_ops import lower_movement, node_kind + + if node_kind(node) == "identity_move": + return lower_movement(node, registry, registry.extra_operations) + + if kind in ("identity", "dropout", "getitem"): + if kind == "dropout" and len(node.args) > 2 and bool(node.args[2]): + raise ValueError(f"Training-mode dropout cannot be lowered: {node.name}") + if kind == "getitem": + raise ValueError(f"Unsupported tuple access {node.name}") + source_id = registry.get_or_create(node.args[0]) + quantization = registry.node_to_qparams.get(node) + tensor = registry.tensors[source_id] + if quantization is not None and quantization != ( + tensor.quant_scale, + tensor.quant_zero_point, + ): + raise ValueError(f"Identity op {node.name} changes quantization parameters") + registry.node_to_id[node] = source_id + registry.fused_nodes.add(node) + return None + + if kind == "stack": + from .int8_ops import lower_cat + + return lower_cat(node, registry, registry.extra_operations, stack=True) + + if kind == "cat": + try: + return _lower_int8_concat(node, registry) + except ValueError: + from .int8_ops import lower_cat + + return lower_cat(node, registry, registry.extra_operations) + + if kind in ("flatten", "reshape"): + from .int8_ops import ( + _skip_quantization, + is_movement_terminal, + lower_movement, + MOVEMENT_KINDS, + node_kind, + ) + + if not is_movement_terminal(node): + registry.fused_nodes.add(node) + return None + _require_output_qparams(registry, node) + producer = _skip_quantization(node.args[0]) + result_shape = _shape_from_node(node) + if ( + node_kind(producer) in MOVEMENT_KINDS + or len(result_shape) != 2 + or result_shape[0] != 1 + ): + return lower_movement(node, registry, registry.extra_operations) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + return lower_movement(node, registry, registry.extra_operations) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.FLATTEN, + inputs=[input_id], + outputs=[output_id], + ) + + if kind == "linear": + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_shape = _shape_from_node(node.args[0]) + if len(input_shape) == 4 or len(input_shape) < 1: + raise ValueError( + f"linear {node.name} needs a rank 1 to 3 input, got {input_shape}" + ) + input_id = registry.get_or_create(node.args[0]) + weight_scales = _per_channel_weight_scales(registry, node.args[1]) + weight_id = registry.get_or_create(node.args[1]) + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + node.args[2] if len(node.args) > 2 else None, + input_tensor.quant_scale, + weight_scales, + ) + output_id = registry.get_or_create(output_node) + if activation is not None: + registry.fused_nodes.add(activation) + output_tensor = registry.tensors[output_id] + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack(INT8_PAIR_PARAMS, act_min, act_max) + ( + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.LINEAR, + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + return None + + +def _unary_function(node, kind: str): + import torch.nn.functional as F + + args = list(node.args) + kwargs = dict(node.kwargs) + if kind == "hardswish": + return F.hardswish + if kind == "hardsigmoid": + return F.hardsigmoid + if kind == "sigmoid": + return torch.sigmoid + if kind == "silu": + return F.silu + if kind == "tanh": + return torch.tanh + if kind == "gelu": + approximate = kwargs.get("approximate", args[1] if len(args) > 1 else "none") + return lambda x: F.gelu(x, approximate=approximate) + if kind == "elu": + alpha = float(args[1]) if len(args) > 1 else float(kwargs.get("alpha", 1.0)) + scale = float(args[2]) if len(args) > 2 else float(kwargs.get("scale", 1.0)) + input_scale = ( + float(args[3]) if len(args) > 3 else float(kwargs.get("input_scale", 1.0)) + ) + return lambda x: torch.ops.aten.elu(x, alpha, scale, input_scale) + if kind == "leaky_relu": + slope = ( + float(args[1]) + if len(args) > 1 + else float(kwargs.get("negative_slope", 0.01)) + ) + return lambda x: F.leaky_relu(x, slope) + if kind == "relu": + return torch.relu + if kind == "relu6": + return lambda x: torch.clamp(x, 0.0, 6.0) + if kind in ("hardtanh", "hardtanh_"): + low = float(args[1]) if len(args) > 1 else -1.0 + high = float(args[2]) if len(args) > 2 else 1.0 + return lambda x: torch.clamp(x, low, high) + raise ValueError(f"No lookup-table function for {node.name}") + + +def build_lut(function, source: XTS3Tensor, result: XTS3Tensor) -> bytes: + codes = torch.arange(-128, 128, dtype=torch.int32).to(torch.int8) + values = torch.ops.quantized_decomposed.dequantize_per_tensor( + codes, source.quant_scale, source.quant_zero_point, -128, 127, torch.int8 + ) + outputs = function(values.to(torch.float32)).to(torch.float32) + if not torch.all(torch.isfinite(outputs)): + raise ValueError("Lookup-table function produced non-finite values") + quantized = torch.ops.quantized_decomposed.quantize_per_tensor( + outputs, result.quant_scale, result.quant_zero_point, -128, 127, torch.int8 + ) + return quantized.contiguous().numpy().tobytes() + + +def _lut_operation( + node, input_id: int, output_id: int, registry: TensorRegistry, function=None +): + kind = _target_kind(node.target) + function = function or _unary_function(node, kind) + table = build_lut(function, registry.tensors[input_id], registry.tensors[output_id]) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.LUT, + inputs=[input_id], + outputs=[output_id], + params=table, + ) + + +def _constant_scalar(node, registry: TensorRegistry): + if isinstance(node, (int, float)): + return float(node) + if not hasattr(node, "op"): + return None + value = None + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized = registry.resolver.resolve(node.args[0]) + if quantized is not None: + value = torch.ops.quantized_decomposed.dequantize_per_tensor( + quantized, + float(node.args[1]), + int(node.args[2]), + int(node.args[3]), + int(node.args[4]), + quantized.dtype, + ) + elif node.op in ("placeholder", "get_attr"): + value = registry.resolver.resolve(node) + if value is None or value.numel() != 1: + return None + return float(value.reshape(-1)[0]) + + +def _reject_constant_operands(node, registry: TensorRegistry) -> None: + for arg in node.args[:2]: + if not hasattr(arg, "op"): + continue + source = ( + arg.args[0] + if (arg.op == "call_function" and "dequantize_per" in str(arg.target)) + else arg + ) + if hasattr(source, "op") and registry.resolver.resolve(source) is not None: + raise ValueError( + f"{node.name} uses a constant tensor operand; only scalar constants are supported" + ) + + +def _lower_scalar_binary(node, kind: str, registry: TensorRegistry): + lhs, rhs = node.args[0], node.args[1] + constant = _constant_scalar(rhs, registry) + tensor_arg = lhs + if constant is None: + constant = _constant_scalar(lhs, registry) + tensor_arg = rhs + if constant is None: + return None + alpha = float(node.kwargs.get("alpha", 1.0)) if kind == "add" else 1.0 + _require_output_qparams(registry, node) + input_id = registry.get_or_create(tensor_arg) + output_id = registry.get_or_create(node) + tensor_on_left = tensor_arg is lhs + + def function(x): + if kind != "add": + return x * constant + return x + constant * alpha if tensor_on_left else constant + alpha * x + + return _lut_operation(node, input_id, output_id, registry, function) + + +def _lower_int8_mul(node, registry: TensorRegistry) -> XTS3Operation: + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + from .int8_ops import broadcast_operand + + lhs_id = registry.get_or_create(node.args[0]) + rhs_id = registry.get_or_create(node.args[1]) + output_id = registry.get_or_create(output_node) + if activation is not None: + registry.fused_nodes.add(activation) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + result = registry.tensors[output_id] + + def per_channel_pattern(full, small): + return ( + len(full.shape) == 4 + and len(small.shape) == 4 + and full.shape == result.shape + and small.shape == (1, full.shape[1], 1, 1) + and full.shape[0] == 1 + and not small.is_constant + ) + + if not (lhs.shape == rhs.shape == result.shape) and not ( + per_channel_pattern(lhs, rhs) or per_channel_pattern(rhs, lhs) + ): + lhs_id = broadcast_operand( + node.args[0], result.shape, registry, registry.extra_operations + ) + rhs_id = broadcast_operand( + node.args[1], result.shape, registry, registry.extra_operations + ) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + if lhs.shape == rhs.shape == result.shape: + mode = MUL_MODE_ELEMENTWISE + inputs = [lhs_id, rhs_id] + else: + + def per_channel(full, small): + return ( + len(full.shape) == 4 + and len(small.shape) == 4 + and full.shape == result.shape + and small.shape == (1, full.shape[1], 1, 1) + and full.shape[0] == 1 + ) + + if per_channel(lhs, rhs): + inputs = [lhs_id, rhs_id] + elif per_channel(rhs, lhs): + inputs = [rhs_id, lhs_id] + else: + raise ValueError( + f"mul {node.name} broadcasts {lhs.shape} x {rhs.shape}; only identical shapes " + "or (1,C,H,W) x (1,C,1,1) are supported" + ) + mode = MUL_MODE_CHANNEL_BROADCAST + multiplier, shift = _quantize_multiplier( + lhs.quant_scale * rhs.quant_scale / result.quant_scale + ) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.MUL, + inputs=inputs, + outputs=[output_id], + params=struct.pack(INT8_MUL_PARAMS, multiplier, shift, act_min, act_max, mode), + ) + + +def build_softmax_table(input_scale: float) -> bytes: + import math + + one = 1 << SOFTMAX_EXP_BITS + table = [ + int(math.floor(math.exp(-d * input_scale) * one + 0.5)) for d in range(256) + ] + return struct.pack("<256I", *table) + + +def _lower_int8_softmax(node, registry: TensorRegistry) -> XTS3Operation: + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + rank = len(source.shape) + dim = int(node.args[1]) if len(node.args) > 1 else -1 + if dim < 0: + dim += rank + if source.shape[0] != 1 or not ( + (rank == 2 and dim == 1) or (rank == 4 and dim == 1) + ): + raise ValueError( + f"Only channel softmax on batch-1 rank 2 or 4 tensors is supported: {node.name}" + ) + if ( + abs(result.quant_scale - SOFTMAX_OUTPUT_SCALE) > 1e-9 + or result.quant_zero_point != SOFTMAX_OUTPUT_ZERO_POINT + ): + raise ValueError( + f"softmax {node.name} output must use scale 1/256 and zero point -128 " + "(quantize with EspNNQuantizer)" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.SOFTMAX, + inputs=[input_id], + outputs=[output_id], + params=build_softmax_table(source.quant_scale), + ) + + +def _pool_window(value, name: str) -> tuple[int, int]: + if value is None or (isinstance(value, (list, tuple)) and len(value) == 0): + return None + if isinstance(value, int): + return (int(value), int(value)) + values = tuple(int(v) for v in value) + if len(values) == 1: + return (values[0], values[0]) + if len(values) != 2: + raise ValueError(f"Expected a 2D {name}, got {values}") + return values + + +def _lower_int8_pool( + node, kind: str, registry: TensorRegistry, source_node=None +) -> XTS3Operation: + source_node = source_node if source_node is not None else node + args = list(source_node.args) + kernel = _pool_window(args[1], "kernel_size") + stride = _pool_window(args[2] if len(args) > 2 else None, "stride") or kernel + padding = _pool_window(args[3] if len(args) > 3 else 0, "padding") or (0, 0) + if kind == "max_pool2d": + dilation = _pool_window(args[4] if len(args) > 4 else 1, "dilation") or (1, 1) + ceil_mode = bool(args[5]) if len(args) > 5 else False + if dilation != (1, 1): + raise ValueError(f"Dilated max pooling is not supported: {node.name}") + opcode = XTS3OpCode.MAX_POOL2D + else: + ceil_mode = bool(args[4]) if len(args) > 4 else False + count_include_pad = bool(args[5]) if len(args) > 5 else True + divisor_override = args[6] if len(args) > 6 else None + if divisor_override is not None: + raise ValueError( + f"avg_pool2d divisor_override is not supported: {node.name}" + ) + if count_include_pad and padding != (0, 0): + raise ValueError( + f"avg_pool2d with padding must use count_include_pad=False for ESP-NN: {node.name}" + ) + opcode = XTS3OpCode.AVG_POOL2D + if ceil_mode: + raise ValueError(f"ceil_mode pooling is not supported: {node.name}") + if padding[0] * 2 > kernel[0] or padding[1] * 2 > kernel[1]: + raise ValueError(f"Pooling padding exceeds half the window: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(source_node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError(f"Pooling {node.name} changes quantization parameters") + if len(source.shape) != 4 or source.shape[0] != 1: + raise ValueError(f"Pooling input must be batch-1 rank 4: {node.name}") + return XTS3Operation( + op_id=-1, + opcode=opcode, + inputs=[input_id], + outputs=[output_id], + params=struct.pack( + INT8_POOL_PARAMS, + kernel[0], + kernel[1], + stride[0], + stride[1], + padding[0], + padding[1], + 1, + 1, + -128, + 127, + ), + ) + + +def _lower_adaptive_avg_pool( + node, output_size, registry: TensorRegistry +) -> XTS3Operation: + input_shape = _shape_from_node(node.args[0]) + if len(input_shape) != 4 or len(output_size) != 2: + raise ValueError(f"Adaptive pooling needs a rank-4 input: {node.name}") + height, width = input_shape[2], input_shape[3] + out_h, out_w = output_size + if height % out_h or width % out_w: + raise ValueError( + f"Adaptive pooling {node.name} maps {height}x{width} to {out_h}x{out_w}; " + "only evenly divisible sizes are supported" + ) + kernel = (height // out_h, width // out_w) + if max(kernel) > 255: + raise ValueError(f"Adaptive pooling window exceeds 255: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError( + f"Adaptive pooling {node.name} changes quantization parameters" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.AVG_POOL2D, + inputs=[input_id], + outputs=[output_id], + params=struct.pack( + INT8_POOL_PARAMS, + kernel[0], + kernel[1], + kernel[0], + kernel[1], + 0, + 0, + 1, + 1, + -128, + 127, + ), + ) + + +def _lower_int8_concat(node, registry: TensorRegistry) -> XTS3Operation: + sources = list(node.args[0]) + dim = int(node.args[1]) if len(node.args) > 1 else 0 + if not 2 <= len(sources) <= 6: + raise ValueError(f"Concatenation needs 2 to 6 inputs: {node.name}") + _require_output_qparams(registry, node) + input_ids = [registry.get_or_create(source) for source in sources] + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + rank = len(result.shape) + if dim < 0: + dim += rank + if dim != 1 or rank not in (2, 4) or result.shape[0] != 1: + raise ValueError( + f"Only batch-1 channel concatenation is supported: {node.name}" + ) + for input_id in input_ids: + tensor = registry.tensors[input_id] + if ( + tensor.quant_scale != result.quant_scale + or tensor.quant_zero_point != result.quant_zero_point + ): + raise ValueError( + f"Concatenation inputs of {node.name} use different qparams" + ) + if len(tensor.shape) != rank or tensor.shape[2:] != result.shape[2:]: + raise ValueError( + f"Concatenation inputs of {node.name} have mismatched shapes" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.CONCAT, + inputs=input_ids, + outputs=[output_id], + ) + + +INT8_PASSTHROUGH_MARKERS = ( + "quantize_per_tensor", + "dequantize_per_tensor", + "dequantize_per_channel", + "quantize_per_channel", +) + + +def _is_int8_passthrough(node) -> bool: + text = str(node.target) + return any(marker in text for marker in INT8_PASSTHROUGH_MARKERS) + + +def _quantized_source(node): + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized = node.args[0] + if getattr( + quantized, "op", None + ) == "call_function" and "quantize_per_tensor" in str(quantized.target): + return quantized.args[0] + if ( + node.op == "call_function" + and "quantize_per_tensor" in str(node.target) + and "dequantize" not in str(node.target) + ): + return node.args[0] + return None + + +def _user_input_placeholders(exported_program): + graph_module = exported_program.graph_module + placeholders = [ + node for node in graph_module.graph.nodes if node.op == "placeholder" + ] + signature = getattr(exported_program, "graph_signature", None) + if signature is None: + return placeholders + user_inputs = set(signature.user_inputs) + return [node for node in placeholders if node.name in user_inputs] + + +def _mark_int8_graph_io( + exported_program, registry: TensorRegistry +) -> tuple[list[int], list[int]]: + graph_module = exported_program.graph_module + inputs: list[int] = [] + for node in _user_input_placeholders(exported_program): + if node not in registry.node_to_id: + if registry.resolver.resolve(node) is not None: + continue + raise ValueError( + f"Graph input {node.name} is not consumed by an int8 operation" + ) + tensor = registry.tensors[registry.node_to_id[node]] + if tensor.is_constant: + continue + tensor.flags |= TENSOR_FLAG_GRAPH_INPUT + inputs.append(tensor.tensor_id) + if not inputs: + raise ValueError("Int8 graph has no quantized runtime input") + + outputs: list[int] = [] + for node in graph_module.graph.nodes: + if node.op != "output": + continue + values = node.args[0] + if not isinstance(values, (list, tuple)): + values = (values,) + for value in values: + source = _quantized_source(value) + if source is None or source not in registry.node_to_id: + raise ValueError( + f"Graph output {getattr(value, 'name', value)} is not a dequantized int8 tensor" + ) + registry.tensors[ + registry.node_to_id[source] + ].flags |= TENSOR_FLAG_GRAPH_OUTPUT + outputs.append(registry.node_to_id[source]) + if not outputs: + raise ValueError("Int8 graph has no outputs") + return inputs, outputs + + +def build_graph(exported_program) -> XTS3Graph: + """Lower a PT2E-quantized ExportedProgram (or delegate partition) to an XTS3 int8 graph.""" + graph_module = exported_program.graph_module + registry = TensorRegistry(ExportedTensorResolver(exported_program), graph_module) + operations = [] + for node in graph_module.graph.nodes: + if node.op != "call_function": + continue + lowered = lower_int8_node(node, registry) + for extra in registry.extra_operations: + extra.op_id = len(operations) + operations.append(extra) + registry.extra_operations.clear() + if lowered is None: + if node not in registry.fused_nodes and not _is_int8_passthrough(node): + raise ValueError( + f"Unsupported operation for the ESP-NN int8 backend: {node.name} ({node.target})" + ) + continue + lowered.op_id = len(operations) + operations.append(lowered) + graph_inputs, graph_outputs = _mark_int8_graph_io(exported_program, registry) + return XTS3Graph( + tensors=registry.tensors, + operations=operations, + constant_data=bytes(registry.constant_data), + version=XTS3_VERSION, + tensor_names={ + tensor_id: node.name for node, tensor_id in registry.node_to_id.items() + }, + inputs=graph_inputs, + outputs=graph_outputs, + ) + + +def _serialize_tensor(tensor: XTS3Tensor) -> bytes: + if len(tensor.shape) > 4: + raise ValueError(f"Tensor rank exceeds the XTS3 limit of 4: {tensor.shape}") + dims = list(tensor.shape) + [0] * (4 - len(tensor.shape)) + flags = tensor.flags | (TENSOR_FLAG_QUANTIZED if tensor.quant_scale > 0 else 0) + return struct.pack( + TENSOR_FORMAT, + tensor.tensor_id, + tensor.data_offset, + int(tensor.dtype), + flags, + len(tensor.shape), + *dims, + tensor.quant_scale, + tensor.quant_zero_point, + ) + + +def _serialize_operation(operation: XTS3Operation, params_offset: int) -> bytes: + if len(operation.inputs) > 6: + raise ValueError(f"Operation {operation.op_id} has more than 6 inputs") + if len(operation.outputs) > 2: + raise ValueError(f"Operation {operation.op_id} has more than 2 outputs") + inputs = operation.inputs + [0] * (6 - len(operation.inputs)) + outputs = operation.outputs + [0] * (2 - len(operation.outputs)) + return struct.pack( + OP_FORMAT, + operation.op_id & 0xFF, + int(operation.opcode), + len(operation.inputs), + len(operation.outputs), + *inputs, + *outputs, + params_offset, + len(operation.params), + ) + + +def serialize_graph(graph: XTS3Graph) -> bytes: + """Serialize to the XTS3 wire format; constants start 16-byte aligned.""" + if graph.version not in XTS3_VERSIONS: + raise ValueError(f"Cannot serialize XTS3 version {graph.version}") + tensor_data = b"".join(_serialize_tensor(tensor) for tensor in graph.tensors) + parameter_data = bytearray() + operation_data = bytearray() + for operation in graph.operations: + operation_data += _serialize_operation(operation, len(parameter_data)) + parameter_data += operation.params + io_table = b"" + if graph.version >= 6: + if len(graph.inputs) > 0xFFFF or len(graph.outputs) > 0xFFFF: + raise ValueError("Too many graph inputs or outputs") + io_table = struct.pack( + IO_TABLE_HEADER, len(graph.inputs), len(graph.outputs) + ) + struct.pack( + f"<{len(graph.inputs) + len(graph.outputs)}I", *graph.inputs, *graph.outputs + ) + prefix = ( + HEADER_SIZE + + len(io_table) + + len(tensor_data) + + len(operation_data) + + len(parameter_data) + ) + parameter_data += b"\x00" * ((-prefix) % 16) + header = struct.pack( + HEADER_FORMAT, + MAGIC, + graph.version, + len(graph.tensors), + len(graph.operations), + len(parameter_data), + len(graph.constant_data), + ) + return ( + header + + io_table + + tensor_data + + bytes(operation_data) + + bytes(parameter_data) + + bytes(graph.constant_data) + ) + + +def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 + """Parse and bounds-check an XTS3 v5 or v6 int8 graph.""" + if len(data) < HEADER_SIZE: + raise ValueError("Graph blob is smaller than the header") + magic, version, tensor_count, operation_count, parameter_size, constant_size = ( + struct.unpack_from(HEADER_FORMAT, data, 0) + ) + if magic != MAGIC: + raise ValueError(f"Invalid graph magic: {magic!r}") + if version not in XTS3_VERSIONS: + raise ValueError( + f"Unsupported XTS3 version {version}; expected one of {XTS3_VERSIONS}" + ) + offset = HEADER_SIZE + graph_inputs: list[int] = [] + graph_outputs: list[int] = [] + if version >= 6: + if offset + IO_TABLE_HEADER_SIZE > len(data): + raise ValueError("Graph IO table exceeds blob") + input_count, output_count = struct.unpack_from(IO_TABLE_HEADER, data, offset) + offset += IO_TABLE_HEADER_SIZE + if offset + 4 * (input_count + output_count) > len(data): + raise ValueError("Graph IO table exceeds blob") + ids = struct.unpack_from(f"<{input_count + output_count}I", data, offset) + graph_inputs, graph_outputs = list(ids[:input_count]), list(ids[input_count:]) + offset += 4 * (input_count + output_count) + + if offset + tensor_count * TENSOR_SIZE > len(data): + raise ValueError("Graph tensor region exceeds blob") + tensors = [] + for _ in range(tensor_count): + fields = struct.unpack_from(TENSOR_FORMAT, data, offset) + offset += TENSOR_SIZE + tensor_id, data_offset, dtype_id, flags, rank = fields[:5] + quant_scale, quant_zero_point = fields[9:11] + if rank > 4: + raise ValueError(f"Tensor {tensor_id} has invalid rank {rank}") + try: + dtype = XTS3DType(dtype_id) + except ValueError as exc: + raise ValueError( + f"Tensor {tensor_id} has invalid dtype id {dtype_id}" + ) from exc + shape = tuple(fields[5 : 5 + rank]) + tensors.append( + XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=dtype, + data_offset=data_offset, + data_size=(prod(shape) if shape else 1) * dtype_size(dtype), + flags=flags, + quant_scale=quant_scale, + quant_zero_point=quant_zero_point, + ) + ) + + if offset + operation_count * OP_SIZE > len(data): + raise ValueError("Graph operation region exceeds blob") + raw_operations = [] + for _ in range(operation_count): + fields = struct.unpack_from(OP_FORMAT, data, offset) + offset += OP_SIZE + op_id, opcode, num_inputs, num_outputs = fields[:4] + params_offset, params_size = fields[12:14] + if num_inputs > 6 or num_outputs > 2: + raise ValueError(f"Operation {op_id} has invalid input or output counts") + if params_offset + params_size > parameter_size: + raise ValueError( + f"Operation {op_id} parameters exceed the parameter region" + ) + raw_operations.append( + ( + op_id, + XTS3OpCode(opcode), + list(fields[4:10])[:num_inputs], + list(fields[10:12])[:num_outputs], + params_offset, + params_size, + ) + ) + parameter_base = offset + constant_base = parameter_base + parameter_size + if constant_base + constant_size > len(data): + raise ValueError("Graph constant region exceeds blob") + operations = [ + XTS3Operation( + op_id=op_id, + opcode=opcode, + inputs=inputs, + outputs=outputs, + params=data[ + parameter_base + + params_offset : parameter_base + + params_offset + + params_size + ], + ) + for op_id, opcode, inputs, outputs, params_offset, params_size in raw_operations + ] + if version == 5: + graph_inputs = [ + t.tensor_id for t in tensors if t.flags & TENSOR_FLAG_GRAPH_INPUT + ] + graph_outputs = [ + t.tensor_id for t in tensors if t.flags & TENSOR_FLAG_GRAPH_OUTPUT + ] + for tensor_id in graph_inputs + graph_outputs: + if tensor_id >= len(tensors): + raise ValueError(f"Graph IO table references missing tensor {tensor_id}") + return XTS3Graph( + tensors=tensors, + operations=operations, + constant_data=data[constant_base : constant_base + constant_size], + version=version, + inputs=graph_inputs, + outputs=graph_outputs, + ) + + +__all__ = [ + "MAGIC", + "XTS3_VERSION", + "XTS3_VERSIONS", + "XTS3DType", + "XTS3Graph", + "XTS3OpCode", + "XTS3Operation", + "XTS3Tensor", + "build_graph", + "lower_int8_node", + "parse_graph", + "serialize_graph", +] diff --git a/backends/espressif/test/__init__.py b/backends/espressif/test/__init__.py new file mode 100644 index 00000000000..2d8dd1da1a1 --- /dev/null +++ b/backends/espressif/test/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/backends/espressif/test/test_cpu_fallback.py b/backends/espressif/test/test_cpu_fallback.py new file mode 100644 index 00000000000..c8db1f32f71 --- /dev/null +++ b/backends/espressif/test/test_cpu_fallback.py @@ -0,0 +1,169 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for partial delegation: int8 regions run on the ESP-NN delegate, +unsupported operators stay on the ExecuTorch CPU (portable) kernels. + + PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ + /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_fallback.py +""" + +from __future__ import annotations + +import math + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.lowering import to_espnn_edge + +from executorch.backends.espressif.reference.program_emulator import ProgramEmulator + +from executorch.backends.espressif.test.test_int8_ops import quantize +from executorch.exir.lowered_backend_module import LoweredBackendModule + + +torch.set_grad_enabled(False) + + +def sqnr_db(reference: torch.Tensor, actual: torch.Tensor) -> float: + reference = reference.to(torch.float64) + noise = (reference - actual.to(torch.float64)).pow(2).sum().item() + signal = reference.pow(2).sum().item() + if noise == 0: + return math.inf + return 10 * math.log10(signal / noise) + + +class CumsumHead(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + self.out = nn.Conv2d(8, 4, 1) + + def forward(self, x): + y = F.relu(self.conv(x)) + y = torch.cumsum(y, dim=3) + return self.out(y) + + +class TinyTransformer(nn.Module): + def __init__(self, dim=16, heads=2, classes=5): + super().__init__() + self.heads = heads + self.stem = nn.Conv2d(3, dim, 4, stride=4) + self.norm1 = nn.LayerNorm(dim) + self.qkv = nn.Linear(dim, 3 * dim) + self.proj = nn.Linear(dim, dim) + self.norm2 = nn.LayerNorm(dim) + self.mlp1 = nn.Linear(dim, 2 * dim) + self.mlp2 = nn.Linear(2 * dim, dim) + self.head = nn.Linear(dim, classes) + + def forward(self, x): + tokens = self.stem(x).flatten(2).transpose(1, 2) + n, t, d = tokens.shape + h = self.norm1(tokens) + q, k, v = self.qkv(h).chunk(3, dim=-1) + q = q.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + k = k.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + v = v.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + attention = torch.softmax( + q @ k.transpose(-2, -1) / math.sqrt(d // self.heads), dim=-1 + ) + mixed = (attention @ v).transpose(1, 2).reshape(n, t, d) + tokens = tokens + self.proj(mixed) + tokens = tokens + self.mlp2(F.gelu(self.mlp1(self.norm2(tokens)))) + return self.head(tokens.mean(dim=1)) + + +class KeywordLSTM(nn.Module): + def __init__(self): + super().__init__() + self.frontend = nn.Linear(10, 16) + self.lstm = nn.LSTM(16, 24, batch_first=True) + self.classifier = nn.Linear(24, 6) + + def forward(self, features): + x = F.relu(self.frontend(features)) + sequence, _ = self.lstm(x) + return self.classifier(sequence[:, -1, :]) + + +class CpuIsland(nn.Module): + def __init__(self): + super().__init__() + self.a = nn.Conv2d(3, 8, 3, padding=1) + self.b = nn.Conv2d(8, 8, 3, padding=1) + self.c = nn.Conv2d(8, 4, 1) + + def forward(self, x): + y = F.relu(self.a(x)) + z = torch.nn.functional.normalize(y, dim=1) + z = F.relu(self.b(z)) + y + return self.c(z) + + +torch.manual_seed(31) +CASES = [ + ("cumsum-on-cpu", CumsumHead(), [(1, 3, 8, 8)], 25.0), + ("tiny-transformer", TinyTransformer(), [(1, 3, 16, 16)], 15.0), + ("lstm-keywords", KeywordLSTM(), [(1, 12, 10)], 15.0), + ("cpu-island-between-delegates", CpuIsland(), [(1, 3, 8, 8)], 20.0), +] + + +def lower(model, shapes): + converted, program = quantize(model, shapes) + edge, report = to_espnn_edge(program) + return converted, edge, report + + +@pytest.mark.parametrize( + "name, model, shapes, min_sqnr", CASES, ids=[c[0] for c in CASES] +) +def test_partial_delegation_tracks_pytorch(name, model, shapes, min_sqnr): + converted, edge, report = lower(model, shapes) + program = edge.exported_program() + delegates = [ + m + for _, m in program.graph_module.named_children() + if isinstance(m, LoweredBackendModule) + ] + assert delegates, f"{name}: nothing was delegated" + assert report["cpu_ops"], f"{name}: expected some operators to fall back to the CPU" + emulator = ProgramEmulator(program) + for trial in range(3): + torch.manual_seed(500 + trial) + inputs = [torch.randn(shape) for shape in shapes] + expected = converted(*inputs) + actual = emulator(*inputs) + expected = expected if isinstance(expected, (tuple, list)) else (expected,) + actual = actual if isinstance(actual, (tuple, list)) else (actual,) + for want, got in zip(expected, actual): + assert got.shape == want.shape + assert ( + sqnr_db(want, got) >= min_sqnr + ), f"{name}: SQNR {sqnr_db(want, got):.1f} dB" + assert len(edge.to_executorch().buffer) > 0 + + +def test_fully_supported_model_has_no_cpu_ops(): + from executorch.backends.espressif.test.test_int8_lowering import SEBlockModel + + _, edge, report = lower(SEBlockModel(), [(1, 3, 16, 16)]) + assert report["partitions"] == 1 + assert report["cpu_ops"] == [] + + +def test_strict_mode_still_rejects_unsupported_operators(): + _, program = quantize(CumsumHead(), [(1, 3, 8, 8)]) + with pytest.raises(Exception, match="cumsum"): + to_espnn_edge(program, allow_cpu_fallback=False) diff --git a/backends/espressif/test/test_fold_batch_norm.py b/backends/espressif/test/test_fold_batch_norm.py new file mode 100644 index 00000000000..e1c07632700 --- /dev/null +++ b/backends/espressif/test/test_fold_batch_norm.py @@ -0,0 +1,85 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +import torch +import torch.nn as nn + +from executorch.backends.espressif._passes.fold_batch_norm import ( + count_conv_bn, + fold_batch_norm, +) +from torchvision import models + + +torch.set_grad_enabled(False) + + +class ConvBnStack(nn.Module): + def __init__(self): + super().__init__() + self.features = nn.Sequential( + nn.Conv2d(3, 8, 3, padding=1, bias=False), + nn.BatchNorm2d(8), + nn.ReLU6(), + nn.Conv2d(8, 8, 3, padding=1, groups=8), + nn.BatchNorm2d(8), + nn.ReLU(), + ) + + def forward(self, x): + return self.features(x) + + +def _randomize_batch_norm(model: nn.Module) -> None: + generator = torch.Generator().manual_seed(0) + for module in model.modules(): + if isinstance(module, nn.BatchNorm2d): + module.running_mean.copy_( + torch.randn(module.num_features, generator=generator) * 0.1 + ) + module.running_var.copy_( + torch.rand(module.num_features, generator=generator) + 0.5 + ) + module.weight.copy_( + torch.rand(module.num_features, generator=generator) + 0.5 + ) + module.bias.copy_( + torch.randn(module.num_features, generator=generator) * 0.1 + ) + + +def test_folding_removes_batch_norm_and_preserves_outputs(): + model = ConvBnStack().eval() + _randomize_batch_norm(model) + x = torch.randn(2, 3, 16, 16) + expected = model(x) + folded, count = fold_batch_norm(model, inplace=False) + assert count == 2 + assert count_conv_bn(folded) == (2, 0) + assert count_conv_bn(model) == (2, 2) + assert torch.allclose(folded(x), expected, atol=1e-5) + + +def test_folding_mobilenet_v2_matches_eager(): + model = models.mobilenet_v2(weights=None).eval() + _randomize_batch_norm(model) + convs, norms = count_conv_bn(model) + folded, count = fold_batch_norm(model, inplace=False) + assert count == norms > 0 + assert count_conv_bn(folded) == (convs, 0) + x = torch.randn(1, 3, 96, 96) + assert torch.allclose(folded(x), model(x), atol=1e-4, rtol=1e-4) + + +def test_training_mode_is_rejected(): + model = ConvBnStack().train() + try: + fold_batch_norm(model) + except ValueError: + return + raise AssertionError("fold_batch_norm accepted a model in training mode") diff --git a/backends/espressif/test/test_int8_lowering.py b/backends/espressif/test/test_int8_lowering.py new file mode 100644 index 00000000000..c16ff1409b5 --- /dev/null +++ b/backends/espressif/test/test_int8_lowering.py @@ -0,0 +1,336 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for the ESP32 int8 quantize-and-lower pipeline. + + PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ + /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_int8.py +""" + +from __future__ import annotations + +import random +import struct + +import numpy as np +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization.xts3 import ( + _quantize_multiplier, + build_graph, + parse_graph, + serialize_graph, + XTS3_VERSION, + XTS3OpCode, +) +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +torch.set_grad_enabled(False) + +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 + + +def c_srdhm(a: int, b: int) -> int: + if a == b == INT32_MIN: + return INT32_MAX + nudge = (1 << 30) if (a >= 0) == (b >= 0) else 1 - (1 << 30) + product = a * b + nudge + quotient = abs(product) >> 31 + return quotient if product >= 0 else -quotient + + +def c_rounding_divide(x: int, exponent: int) -> int: + mask = (1 << exponent) - 1 + remainder = x & mask + result = x >> exponent + threshold = (mask >> 1) + (1 if result < 0 else 0) + return result + (1 if remainder > threshold else 0) + + +def c_requantize(x: int, multiplier: int, shift: int) -> int: + left = max(shift, 0) + right = max(-shift, 0) + shifted = ((x << left) + (1 << 31)) % (1 << 32) - (1 << 31) + return c_rounding_divide(c_srdhm(shifted, multiplier), right) + + +def test_requantization_matches_c_semantics(): + rng = random.Random(0) + xs, mults, shifts = [], [], [] + for _ in range(20000): + xs.append(rng.randint(-(1 << 24), 1 << 24)) + mults.append(rng.randint(1 << 30, INT32_MAX)) + shifts.append(rng.randint(-20, 2)) + xs += [INT32_MIN, INT32_MAX, 0, -1, 1] + mults += [INT32_MAX, 1 << 30, 1 << 30, INT32_MAX, INT32_MAX] + shifts += [0, -1, -31, -5, 0] + expected = [c_requantize(x, m, s) for x, m, s in zip(xs, mults, shifts)] + actual = reference.multiply_by_quantized_multiplier( + np.array(xs, dtype=np.int64), np.array(mults), np.array(shifts) + ) + assert actual.tolist() == expected + + +def test_srdhm_overflow_case(): + assert ( + int(reference.saturating_rounding_doubling_high_mul(INT32_MIN, INT32_MIN)) + == INT32_MAX + ) + + +@pytest.mark.parametrize("value", [1e-6, 0.0123, 0.5, 0.999999, 1.0, 3.75, 1e3]) +def test_quantize_multiplier_reconstructs_value(value): + multiplier, shift = _quantize_multiplier(value) + assert (1 << 30) <= multiplier <= INT32_MAX + assert abs(multiplier * 2.0 ** (shift - 31) - value) <= value * 2**-30 + + +class ConvActModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.dw = nn.Conv2d(16, 16, 3, padding=1, groups=16) + self.grouped = nn.Conv2d(16, 16, 3, padding=1, groups=4) + self.pw = nn.Conv2d(16, 8, 1) + self.fc = nn.Linear(8 * 2 * 2, 10) + + def forward(self, x): + x = F.relu6(self.conv(x)) + y = F.relu(self.dw(x)) + x = F.relu(self.grouped(y) + x) + x = F.max_pool2d(x, 3, 2, 1) + x = self.pw(x) + x = F.avg_pool2d(F.relu(x), 2) + return self.fc(torch.flatten(x, 1)) + + +class ConcatModel(nn.Module): + def __init__(self): + super().__init__() + self.a = nn.Conv2d(4, 6, 1) + self.b = nn.Conv2d(4, 10, 3, padding=1) + self.head = nn.Linear(16, 5) + + def forward(self, x): + x = torch.cat([F.relu(self.a(x)), F.relu(self.b(x))], dim=1) + x = F.adaptive_avg_pool2d(x, 1) + return self.head(torch.flatten(x, 1)) + + +class SEBlockModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.act = nn.Hardswish() + self.fc1 = nn.Conv2d(16, 8, 1) + self.fc2 = nn.Conv2d(8, 16, 1) + self.gate = nn.Hardsigmoid() + self.dw = nn.Conv2d(16, 16, 3, padding=1, groups=16) + self.head = nn.Linear(16, 6) + + def forward(self, x): + x = self.act(self.conv(x)) + scale = self.gate(self.fc2(F.relu(self.fc1(F.adaptive_avg_pool2d(x, 1))))) + x = x * scale + x = F.silu(self.dw(x)) + x = torch.sigmoid(x) * x + x * 0.5 + 1.0 + x = torch.flatten(F.adaptive_avg_pool2d(x, 1), 1) + return torch.softmax(self.head(x), dim=1) + + +class UnsupportedModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 4, 3) + + def forward(self, x): + return torch.cumsum(self.conv(x), dim=1) + + +def quantize(model: nn.Module, shape, samples: int = 16, seed: int = 0): + torch.manual_seed(seed) + model = model.eval() + example = (torch.randn(shape),) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(samples): + prepared(torch.randn(shape)) + converted = convert_pt2e(prepared) + return converted, torch.export.export(converted, example, strict=True) + + +def run_device_model(graph, x: torch.Tensor): + interpreter = reference.ReferenceInterpreter(graph) + source = graph.tensors[interpreter.inputs[0]] + result = graph.tensors[interpreter.outputs[0]] + quantized = reference.to_device_layout( + source, reference.quantize(source, x.numpy()) + ) + output = interpreter.run({source.tensor_id: quantized})[result.tensor_id] + return result, output + + +@pytest.mark.parametrize( + "model, shape", + [ + (ConvActModel(), (1, 3, 16, 16)), + (ConcatModel(), (1, 4, 6, 6)), + (SEBlockModel(), (1, 3, 16, 16)), + ], + ids=["conv-act-pool-linear", "concat-mean-linear", "se-lut-mul-softmax"], +) +def test_lowering_tracks_pytorch_int8(model, shape): + converted, program = quantize(model, shape) + graph = parse_graph(serialize_graph(build_graph(program))) + assert graph.version == XTS3_VERSION + for seed in range(4): + torch.manual_seed(100 + seed) + x = torch.randn(shape) + result, output = run_device_model(graph, x) + expected = converted(x).reshape(-1).numpy() + expected_q = np.clip( + np.rint(expected / result.quant_scale) + result.quant_zero_point, -128, 127 + ) + assert np.max(np.abs(expected_q - output.reshape(-1).astype(np.int64))) <= 2 + + +def test_relu6_is_fused_into_convolution(): + _, program = quantize(ConvActModel(), (1, 3, 16, 16)) + graph = build_graph(program) + opcodes = [op.opcode for op in graph.operations] + assert XTS3OpCode.RELU6 not in opcodes + conv = graph.operations[0] + output = graph.tensors[conv.outputs[0]] + act_min, act_max = struct.unpack_from("= 5 + assert counts.get(XTS3OpCode.SOFTMAX) == 1 + modes = [ + struct.unpack("<5i", op.params)[4] + for op in graph.operations + if op.opcode == XTS3OpCode.MUL + ] + assert 1 in modes and 0 in modes + softmax = [op for op in graph.operations if op.opcode == XTS3OpCode.SOFTMAX][0] + output = graph.tensors[softmax.outputs[0]] + assert output.quant_zero_point == -128 and abs(output.quant_scale - 1 / 256) < 1e-12 + + +def test_lut_matches_pytorch_quantized_function(): + from executorch.backends.espressif.serialization.xts3 import ( + build_lut, + XTS3DType, + XTS3Tensor, + ) + + source = XTS3Tensor( + 0, (1, 4), XTS3DType.INT8, quant_scale=0.05, quant_zero_point=-3 + ) + result = XTS3Tensor( + 1, (1, 4), XTS3DType.INT8, quant_scale=0.03, quant_zero_point=-100 + ) + table = np.frombuffer(build_lut(F.hardswish, source, result), dtype=np.int8) + codes = torch.arange(-128, 128).to(torch.int8) + expected = torch.ops.quantized_decomposed.quantize_per_tensor( + F.hardswish( + torch.ops.quantized_decomposed.dequantize_per_tensor( + codes, 0.05, -3, -128, 127, torch.int8 + ) + ), + 0.03, + -100, + -128, + 127, + torch.int8, + ) + assert table.tolist() == expected.tolist() + + +def test_float_model_cannot_be_lowered_to_int8(): + program = torch.export.export( + ConcatModel().eval(), (torch.randn(1, 4, 6, 6),), strict=True + ) + with pytest.raises(ValueError): + build_graph(program) + + +def test_executorch_delegate_matches_standalone_blob(): + from executorch.backends.espressif.partition.espnn_partitioner import ( + EspNNPartitioner, + ) + from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower + from executorch.exir.lowered_backend_module import LoweredBackendModule + + _, program = quantize(SEBlockModel(), (1, 3, 16, 16)) + standalone = serialize_graph(build_graph(program)) + edge = to_edge_transform_and_lower( + program, + partitioner=[EspNNPartitioner(allow_cpu_fallback=False)], + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + delegates = [ + module + for _, module in edge.exported_program().graph_module.named_children() + if isinstance(module, LoweredBackendModule) + ] + assert len(delegates) == 1 + assert bytes(delegates[0].processed_bytes) == standalone + assert len(edge.to_executorch().buffer) > len(standalone) + + +def test_memory_plan_reuses_buffers(): + _, program = quantize(ConvActModel(), (1, 3, 16, 16)) + graph = build_graph(program) + arena, offsets = reference.plan_activation_arena(graph) + total = sum( + (int(np.prod(t.shape)) + 15) // 16 * 16 + for t in graph.tensors + if not t.is_constant + ) + assert 0 < arena < total + assert all(offset % 16 == 0 for offset in offsets.values()) diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py new file mode 100644 index 00000000000..469282a7a39 --- /dev/null +++ b/backends/espressif/test/test_int8_ops.py @@ -0,0 +1,299 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for the XTS3 v6 additions: data movement (COPY), upsampling, +transposed and dilated convolution, broadcasting, padding, any-axis concat, +split, and multiple graph inputs and outputs. + + PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ + /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_ops_v6.py +""" + +from __future__ import annotations + +import numpy as np +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization.xts3 import ( + build_graph, + parse_graph, + serialize_graph, + XTS3_VERSION, + XTS3OpCode, +) +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +torch.set_grad_enabled(False) + + +def quantize(model: nn.Module, shapes, samples: int = 24, seed: int = 0): + torch.manual_seed(seed) + model = model.eval() + example = tuple(torch.randn(shape) for shape in shapes) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(samples): + prepared(*(torch.randn(shape) for shape in shapes)) + converted = convert_pt2e(prepared) + return converted, torch.export.export(converted, example, strict=True) + + +def lower(program): + graph = parse_graph(serialize_graph(build_graph(program))) + assert graph.version == XTS3_VERSION + return graph + + +def run_reference(graph, inputs): + interpreter = reference.ReferenceInterpreter(graph) + device_inputs = [] + for tensor_id, value in zip(interpreter.inputs, inputs): + tensor = graph.tensors[tensor_id] + device_inputs.append( + reference.to_device_layout( + tensor, reference.quantize(tensor, value.numpy()) + ) + ) + outputs = interpreter.run_outputs(device_inputs) + return [ + (graph.tensors[tensor_id], output) + for tensor_id, output in zip(interpreter.outputs, outputs) + ] + + +def max_int8_error(converted, graph, shapes, trials: int = 4) -> int: + worst = 0 + for trial in range(trials): + torch.manual_seed(1000 + trial) + inputs = [torch.randn(shape) for shape in shapes] + expected = converted(*inputs) + if isinstance(expected, torch.Tensor): + expected = (expected,) + results = run_reference(graph, inputs) + assert len(results) == len(expected) + for (tensor, device_output), want in zip(results, expected): + logical = reference.to_logical_layout(tensor, device_output).astype( + np.int64 + ) + assert logical.shape == tuple(want.shape) + want_q = np.clip( + np.rint(want.numpy() / tensor.quant_scale) + tensor.quant_zero_point, + -128, + 127, + ) + worst = max(worst, int(np.max(np.abs(want_q - logical)))) + return worst + + +class DetectionHead(nn.Module): + def __init__(self, anchors=3, classes=5): + super().__init__() + self.anchors, self.classes = anchors, classes + self.backbone = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.p4 = nn.Conv2d(16, 16, 3, stride=2, padding=1) + self.head3 = nn.Conv2d(16, anchors * (classes + 4), 1) + self.head4 = nn.Conv2d(16, anchors * (classes + 4), 1) + + def _flatten(self, x): + n, _, h, w = x.shape + x = x.view(n, self.anchors, self.classes + 4, h, w) + return x.permute(0, 1, 3, 4, 2).reshape(n, -1, self.classes + 4) + + def forward(self, x): + f3 = F.relu(self.backbone(x)) + f4 = F.relu(self.p4(f3)) + boxes = torch.cat( + [self._flatten(self.head3(f3)), self._flatten(self.head4(f4))], dim=1 + ) + return boxes[..., :4], boxes[..., 4:] + + +class SegmentationDecoder(nn.Module): + def __init__(self): + super().__init__() + self.encoder = nn.Conv2d(3, 8, 3, stride=2, padding=1) + self.dilated = nn.Conv2d(8, 8, 3, padding=2, dilation=2) + self.dilated_dw = nn.Conv2d(8, 8, 3, padding=3, dilation=3, groups=8) + self.up = nn.ConvTranspose2d(8, 6, 3, stride=2, padding=1, output_padding=1) + self.skip = nn.Conv2d(3, 6, 1) + + def forward(self, x): + y = F.relu(self.encoder(x)) + y = F.relu(self.dilated(y)) + y = F.relu(self.dilated_dw(y)) + y = self.up(y) + return F.relu(y + self.skip(x)) + + +class UpsampleModel(nn.Module): + def __init__(self, mode: str, align_corners=None, scale=2.0, size=None): + super().__init__() + self.mode, self.align_corners, self.scale, self.size = ( + mode, + align_corners, + scale, + size, + ) + self.conv = nn.Conv2d(3, 6, 3, stride=2, padding=1) + self.out = nn.Conv2d(6, 4, 1) + + def forward(self, x): + y = F.relu(self.conv(x)) + if self.size is not None: + y = F.interpolate( + y, size=self.size, mode=self.mode, align_corners=self.align_corners + ) + else: + y = F.interpolate( + y, + scale_factor=self.scale, + mode=self.mode, + align_corners=self.align_corners, + ) + return self.out(y) + + +class BroadcastPadSplit(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(4, 8, 3, padding=1) + self.row_gate = nn.Conv2d(4, 8, (3, 1), padding=(1, 0)) + self.register_buffer( + "bias_map", torch.linspace(-1, 1, 8 * 6).reshape(1, 8, 1, 6) + ) + self.mix = nn.Conv2d(4, 4, 1) + + def forward(self, x): + y = self.conv(x) + gate = F.adaptive_avg_pool2d(self.row_gate(x), (1, 6)) + y = F.relu(y + gate) + y = y * torch.sigmoid(gate) + y = y + self.bias_map + a, b = torch.split(y, [4, 4], dim=1) + a = F.pad(a, (1, 1, 2, 0)) + b = F.pad(b, (1, 1, 2, 0)) + return self.mix(torch.cat([a, b], dim=2)) + + +class TwoInputTwoOutput(nn.Module): + def __init__(self): + super().__init__() + self.image = nn.Conv2d(3, 8, 3, stride=2, padding=1) + self.extra = nn.Linear(6, 8) + self.classifier = nn.Linear(8, 4) + self.mask = nn.Conv2d(8, 1, 1) + + def forward(self, image, features): + x = F.relu(self.image(image)) + y = F.relu(self.extra(features)) + pooled = torch.flatten(F.adaptive_avg_pool2d(x, 1), 1) + logits = self.classifier(pooled + y) + mask = torch.sigmoid(self.mask(x)) + return mask, logits + + +class TransposeSelect(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 6, 3, padding=1) + self.fc = nn.Linear(6, 5) + + def forward(self, x): + y = F.relu(self.conv(x)) + tokens = y.flatten(2).transpose(1, 2) + first = tokens[:, 0, :] + last = tokens.select(1, -1) + mixed = torch.stack([first, last], dim=1) + return self.fc(mixed).unsqueeze(-1) + + +torch.manual_seed(20261014) +CASES = [ + ("detection-head-multi-output", DetectionHead(), [(1, 3, 16, 16)], 2), + ("segmentation-dilated-transposed", SegmentationDecoder(), [(1, 3, 16, 16)], 3), + ("upsample-nearest-x2", UpsampleModel("nearest"), [(1, 3, 16, 16)], 2), + ( + "upsample-bilinear-x2", + UpsampleModel("bilinear", align_corners=False), + [(1, 3, 16, 16)], + 2, + ), + ( + "upsample-bilinear-align", + UpsampleModel("bilinear", align_corners=True, size=(13, 11)), + [(1, 3, 16, 16)], + 2, + ), + ("broadcast-pad-split-cat", BroadcastPadSplit(), [(1, 4, 6, 6)], 2), + ("two-inputs-two-outputs", TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)], 2), + ("transpose-select-stack", TransposeSelect(), [(1, 3, 4, 5)], 2), +] + + +@pytest.mark.parametrize( + "name, model, shapes, tolerance", CASES, ids=[c[0] for c in CASES] +) +def test_v6_lowering_tracks_pytorch_int8(name, model, shapes, tolerance): + converted, program = quantize(model, shapes) + graph = lower(program) + assert len(graph.inputs) == len(shapes) + error = max_int8_error(converted, graph, shapes) + assert error <= tolerance, f"{name}: worst int8 difference {error}" + + +def test_detection_head_uses_copy_and_no_rank5_tensors(): + _, program = quantize(DetectionHead(), [(1, 3, 16, 16)]) + graph = lower(program) + assert any(op.opcode == XTS3OpCode.COPY for op in graph.operations) + assert all(len(t.shape) <= 4 for t in graph.tensors) + assert len(graph.outputs) == 2 + + +def test_io_table_round_trips_order(): + _, program = quantize(TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)]) + graph = build_graph(program) + parsed = parse_graph(serialize_graph(graph)) + assert parsed.inputs == graph.inputs and parsed.outputs == graph.outputs + assert [parsed.tensors[i].shape for i in parsed.inputs] == [(1, 3, 12, 12), (1, 6)] + assert [parsed.tensors[i].shape for i in parsed.outputs] == [(1, 1, 6, 6), (1, 4)] + + +def test_constants_are_16_byte_aligned(): + _, program = quantize(BroadcastPadSplit(), [(1, 4, 6, 6)]) + blob = serialize_graph(build_graph(program)) + parsed = parse_graph(blob) + constant_base = len(blob) - len(parsed.constant_data) + assert constant_base % 16 == 0 + + +def test_constant_records_match_their_device_layout(): + """Regression: transposed-conv weights were stored (Cout,kh,kw,Cin) but recorded as (Cin,Cout,kh,kw).""" + _, program = quantize(SegmentationDecoder(), [(1, 3, 16, 16)]) + graph = lower(program) + transposed = [ + op for op in graph.operations if op.opcode == XTS3OpCode.TRANSPOSE_CONV2D + ] + assert transposed + for op in transposed: + weight = graph.tensors[op.inputs[1]] + source = graph.tensors[op.inputs[0]] + result = graph.tensors[op.outputs[0]] + assert weight.shape[0] == result.shape[1] and weight.shape[3] == source.shape[1] + for tensor in graph.tensors: + if tensor.is_constant: + assert tensor.data_size == int(np.prod(tensor.shape)) * ( + 4 if tensor.dtype == 4 else 1 + ) diff --git a/examples/espressif/.gitignore b/examples/espressif/.gitignore new file mode 100644 index 00000000000..771b12ba266 --- /dev/null +++ b/examples/espressif/.gitignore @@ -0,0 +1,10 @@ +build/ +build-*/ +managed_components/ +dependencies.lock +sdkconfig +sdkconfig.old +__pycache__/ +espnn_runner/bundles/ +camera_stream/models/ +benchmarks/work/ diff --git a/examples/espressif/aot_espnn_compiler.py b/examples/espressif/aot_espnn_compiler.py new file mode 100644 index 00000000000..ccc7137387b --- /dev/null +++ b/examples/espressif/aot_espnn_compiler.py @@ -0,0 +1,518 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Quantize a PyTorch model to int8 and lower it for Espressif chips with the +ESP-NN backend, producing a bundle that examples/espressif/espnn_runner can +flash and verify on the board. + +Modes: + executorch (default) ExecuTorch program (.pte): int8 regions run in the + EspNNBackend delegate, anything else on the portable + CPU kernels. Any number of inputs and outputs. + standalone The whole model as one XTS3 int8 graph for the + espnn_runtime interpreter without the ExecuTorch + runtime (smallest firmware). Every operator must lower. + +Examples: + python examples/espressif/aot_espnn_compiler.py --model mobilenet_v2 \ + --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val --output mv2.xtp + python examples/espressif/aot_espnn_compiler.py --model my_models.py:make_model \ + --input-shape 1,3,96,96 --input-shape 1,10 --mode executorch --output mine.xtp + python examples/espressif/aot_espnn_compiler.py --model mobilenet_v3_small --mode standalone \ + --input-shape 1,3,96,96 --output mv3s_96.xtb +""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import math +import random +import sys +import time +from collections import Counter +from pathlib import Path + +import numpy as np +import torch + +from executorch.backends.espressif._passes import fold_batch_norm, remove_dropout +from executorch.backends.espressif.quantizer import EspNNQuantizer +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from espnn_bundle import sqnr_db, write_bundle, xte1_vectors, xtv2_vectors # noqa: E402 + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} +IMAGENETTE_WNID_TO_IMAGENET = { + "n01440764": 0, + "n02102040": 217, + "n02979186": 482, + "n03000684": 491, + "n03028079": 497, + "n03394916": 566, + "n03417042": 569, + "n03425413": 571, + "n03445777": 574, + "n03888257": 701, +} + + +def load_model(spec: str, weights: str) -> torch.nn.Module: + if ":" in spec and spec.split(":", 1)[0].endswith(".py"): + path, factory = spec.split(":", 1) + module_spec = importlib.util.spec_from_file_location("espnn_user_model", path) + if module_spec is None or module_spec.loader is None: + raise ValueError(f"Unable to import {path}") + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_spec.name] = module + module_spec.loader.exec_module(module) + return getattr(module, factory)() + from torchvision import models + + builder = getattr(models, spec, None) + if builder is None: + raise ValueError( + f"Unknown torchvision model {spec}; use path/to/file.py:factory" + ) + return builder(weights=None if weights == "none" else weights) + + +class ImageSource: + def __init__(self, shape, resize: int, mean, std, label_map): + from torchvision import transforms + + height, width = shape[2], shape[3] + self.transform = transforms.Compose( + [ + transforms.Resize(resize or min(height, width)), + transforms.CenterCrop((height, width)), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ] + ) + self.label_map = dict(IMAGENETTE_WNID_TO_IMAGENET) + if label_map: + self.label_map.update(json.loads(Path(label_map).read_text())) + + def load(self, directory: Path, count: int, seed: int): + from PIL import Image + + files = sorted( + p for p in Path(directory).rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES + ) + if not files: + raise ValueError(f"No images found under {directory}") + random.Random(seed).shuffle(files) + return [ + ( + (self.transform(Image.open(path).convert("RGB")).unsqueeze(0),), + int(self.label_map.get(path.parent.name, -1)), + ) + for path in files[:count] + ] + + +def load_npy(path: Path, count: int, shape): + array = np.load(path) + if tuple(array.shape[1:]) != tuple(shape[1:]): + raise ValueError( + f"{path} has sample shape {array.shape[1:]}, expected {shape[1:]}" + ) + return [ + ((torch.from_numpy(array[i : i + 1]).float(),), -1) + for i in range(min(count, len(array))) + ] + + +def make_data(args, shapes): + image_input = len(shapes) == 1 and len(shapes[0]) == 4 and shapes[0][1] == 3 + source = None + if image_input and (args.calib_dir or args.eval_dir): + source = ImageSource( + shapes[0], args.resize, args.mean, args.std, args.label_map + ) + generator = torch.Generator().manual_seed(args.seed) + + def noise(count): + return [ + (tuple(torch.randn(s, generator=generator) for s in shapes), -1) + for _ in range(count) + ] + + if args.calib_npy and len(shapes) == 1: + calibration = load_npy(args.calib_npy, args.calib_samples, shapes[0]) + elif source is not None and args.calib_dir: + calibration = source.load(args.calib_dir, args.calib_samples, args.seed) + else: + print( + " NOTE: no calibration data given; using Gaussian noise (accuracy will be poor)" + ) + calibration = noise(args.calib_samples) + if args.eval_npy and len(shapes) == 1: + evaluation = load_npy(args.eval_npy, args.eval_samples, shapes[0]) + elif source is not None and args.eval_dir: + evaluation = source.load(args.eval_dir, args.eval_samples, args.seed + 1) + else: + evaluation = calibration[: args.eval_samples] + return calibration, evaluation + + +def qat_finetune(model, calibration, args): + """Quantization-aware fine-tuning by KL distillation from the float model (no labels).""" + import torch.nn.functional as F + from torchao.quantization.pt2e import ( + disable_observer, + move_exported_model_to_eval, + move_exported_model_to_train, + ) + from torchao.quantization.pt2e.quantize_pt2e import prepare_qat_pt2e + + data = torch.cat([inputs[0] for inputs, _ in calibration]) + batch = min(args.qat_batch, len(data)) + student = copy.deepcopy(model) + exported = torch.export.export( + student, + (data[:2],), + dynamic_shapes=({0: torch.export.Dim("batch", min=2, max=1024)},), + strict=True, + ).module() + prepared = prepare_qat_pt2e(exported, EspNNQuantizer(is_qat=True)) + for parameter in prepared.parameters(): + if parameter.is_floating_point(): + parameter.requires_grad_(True) + parameters = [p for p in prepared.parameters() if p.requires_grad] + total_steps = max(1, int(round(args.qat_epochs * max(1, len(data) // batch)))) + optimizer = torch.optim.SGD(parameters, lr=args.qat_lr, momentum=0.9) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, total_steps) + generator = torch.Generator().manual_seed(args.seed) + move_exported_model_to_train(prepared) + with torch.no_grad(): + for start in range(0, min(len(data), 16 * batch), batch): + prepared(data[start : start + batch]) + with torch.enable_grad(): + for step in range(total_steps): + if step == int(total_steps * 0.6): + prepared.apply(disable_observer) + inputs = data[torch.randint(0, len(data), (batch,), generator=generator)] + if bool(torch.rand(1, generator=generator) < 0.5): + inputs = inputs.flip(3) + with torch.no_grad(): + teacher = F.softmax(model(inputs), dim=1) + loss = F.kl_div( + F.log_softmax(prepared(inputs), dim=1), teacher, reduction="batchmean" + ) + optimizer.zero_grad() + loss.backward() + optimizer.step() + scheduler.step() + if step % max(1, total_steps // 10) == 0 or step + 1 == total_steps: + print(f" QAT step {step + 1}/{total_steps} loss {loss.item():.4f}") + move_exported_model_to_eval(prepared) + for parameter in prepared.parameters(): + parameter.requires_grad_(False) + return prepared + + +def quantize(model, calibration, args): + example = calibration[0][0] + if args.qat_epochs > 0: + if len(example) != 1: + raise SystemExit("QAT is implemented for single-input models") + print(f"[2/5] PT2E quantization-aware fine-tuning: {args.qat_epochs} epochs") + prepared = qat_finetune(model, calibration, args) + else: + print(f"[2/5] PT2E quantization with {len(calibration)} calibration samples") + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for inputs, _ in calibration: + prepared(*inputs) + quantized = convert_pt2e(prepared) + return quantized, torch.export.export(quantized, example, strict=True) + + +def as_list(value): + return list(value) if isinstance(value, (tuple, list)) else [value] + + +def evaluate(model, quantized, device_fn, evaluation): + totals = Counter() + worst_float = worst_pt2e = math.inf + for inputs, label in evaluation: + float_out = as_list(model(*inputs)) + pt2e_out = as_list(quantized(*inputs)) + device_out = as_list(device_fn(inputs)) + for f, q, d in zip(float_out, pt2e_out, device_out): + worst_float = min(worst_float, sqnr_db(f, d.to(torch.float32))) + worst_pt2e = min(worst_pt2e, sqnr_db(q, d.to(torch.float32))) + if float_out[0].dim() == 2: + float_top1 = int(float_out[0].argmax()) + device_top1 = int(device_out[0].to(torch.float32).argmax()) + totals["samples"] += 1 + totals["top1_agree_with_float"] += int(float_top1 == device_top1) + if label >= 0: + totals["labelled"] += 1 + totals["float_correct"] += int(float_top1 == label) + totals["device_correct"] += int(device_top1 == label) + print( + f" worst output SQNR: device vs float {worst_float:.2f} dB, " + f"device vs PT2E int8 {worst_pt2e:.2f} dB" + ) + if totals["samples"]: + line = f" top-1 agreement with float {totals['top1_agree_with_float']}/{totals['samples']}" + if totals["labelled"]: + line += ( + f"; accuracy float {totals['float_correct']}/{totals['labelled']}, " + f"device {totals['device_correct']}/{totals['labelled']}" + ) + print(line) + return worst_float, worst_pt2e, dict(totals) + + +def compile_standalone(args, model, quantized, program, evaluation): + from executorch.backends.espressif.reference import esp_nn_reference as reference + from executorch.backends.espressif.serialization import ( + build_graph, + parse_graph, + serialize_graph, + ) + + print("[3/5] Lowering to one XTS3 int8 graph (standalone espnn_runtime)") + blob = serialize_graph(build_graph(program)) + graph = parse_graph(blob) + interpreter = reference.ReferenceInterpreter(graph) + ops = Counter(op.opcode.name for op in graph.operations) + arena, _ = reference.plan_activation_arena(graph) + print( + f" {len(graph.operations)} operations {dict(ops)}; blob {len(blob)} bytes; " + f"activation arena {arena} bytes" + ) + + def device_fn(inputs): + device_inputs = [ + reference.to_device_layout( + graph.tensors[i], reference.quantize(graph.tensors[i], x.numpy()) + ) + for i, x in zip(interpreter.inputs, inputs) + ] + outputs = interpreter.run_outputs(device_inputs) + return [ + torch.from_numpy( + reference.dequantize( + graph.tensors[i], reference.to_logical_layout(graph.tensors[i], out) + ) + ) + for i, out in zip(interpreter.outputs, outputs) + ] + + print( + f"[4/5] Evaluating {len(evaluation)} samples: float vs PT2E int8 vs bit-exact device model" + ) + worst_float, worst_pt2e, totals = evaluate(model, quantized, device_fn, evaluation) + samples = [ + [x.numpy() for x in inputs] for inputs, _ in evaluation[: args.vector_samples] + ] + vectors = xtv2_vectors(graph, samples) + report = { + "mode": "standalone", + "operations": dict(ops), + "blob_bytes": len(blob), + "activation_arena_bytes": arena, + "inputs": [list(graph.tensors[i].shape) for i in graph.inputs], + "outputs": [list(graph.tensors[i].shape) for i in graph.outputs], + } + return b"XTB1", blob, vectors, report, worst_float, worst_pt2e, totals + + +def compile_executorch(args, model, quantized, program, evaluation): + from executorch.backends.espressif import to_espnn_edge + from executorch.backends.espressif.reference import ProgramEmulator + from executorch.exir.lowered_backend_module import LoweredBackendModule + + print("[3/5] Lowering through ExecuTorch: EspNN delegates + portable CPU fallback") + edge, partition_report = to_espnn_edge( + program, allow_cpu_fallback=not args.no_cpu_fallback + ) + exported = edge.exported_program() + delegates = [ + m + for _, m in exported.graph_module.named_children() + if isinstance(m, LoweredBackendModule) + ] + pte = edge.to_executorch().buffer + cpu_ops = partition_report["cpu_ops"] + print( + f" {len(delegates)} delegate(s), {partition_report['delegated_ops']} delegated operators, " + f"{len(cpu_ops)} operators on the CPU; .pte {len(pte)} bytes" + ) + for target, count in Counter(op["target"] for op in cpu_ops).most_common(12): + reason = next(op["reason"] for op in cpu_ops if op["target"] == target) + print(f" CPU: {target} x{count} ({reason})") + emulator = ProgramEmulator(exported) + print( + f"[4/5] Evaluating {len(evaluation)} samples: float vs PT2E int8 vs emulated device program" + ) + worst_float, worst_pt2e, totals = evaluate( + model, quantized, lambda inputs: as_list(emulator(*inputs)), evaluation + ) + rows = [ + (list(inputs), as_list(emulator(*inputs))) + for inputs, _ in evaluation[: args.vector_samples] + ] + exact = not cpu_ops + vectors = xte1_vectors( + rows, exact, args.device_min_sqnr, crc=exact and args.crc_outputs + ) + if args.pte_output: + args.pte_output.write_bytes(pte) + report = { + "mode": "executorch", + "pte_bytes": len(pte), + "delegates": len(delegates), + "partitions": partition_report["partitions"], + "delegated_ops": partition_report["delegated_ops"], + "cpu_ops": cpu_ops, + "exact_required": exact, + "min_sqnr_db": args.device_min_sqnr, + "inputs": [list(t.shape) for t in rows[0][0]], + "outputs": [list(t.shape) for t in rows[0][1]], + } + return b"XTP1", pte, vectors, report, worst_float, worst_pt2e, totals + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--model", + required=True, + help="torchvision model name or path/to/file.py:factory", + ) + parser.add_argument( + "--weights", default="DEFAULT", help="torchvision weights name, or 'none'" + ) + parser.add_argument( + "--input-shape", + action="append", + help="input shape such as 1,3,224,224; repeat for several inputs", + ) + parser.add_argument( + "--mode", choices=("executorch", "standalone"), default="executorch" + ) + parser.add_argument( + "--output", type=Path, required=True, help="bundle to write (.xtp or .xtb)" + ) + parser.add_argument( + "--name", help="entry name inside the bundle (default: model name)" + ) + parser.add_argument( + "--pte-output", type=Path, help="executorch mode: also write the plain .pte" + ) + parser.add_argument("--calib-dir", type=Path) + parser.add_argument("--calib-npy", type=Path) + parser.add_argument("--calib-samples", type=int, default=128) + parser.add_argument("--eval-dir", type=Path) + parser.add_argument("--eval-npy", type=Path) + parser.add_argument("--eval-samples", type=int, default=32) + parser.add_argument( + "--vector-samples", + type=int, + default=4, + help="samples embedded in the bundle for on-board checking", + ) + parser.add_argument("--resize", type=int, default=256) + parser.add_argument("--mean", type=float, nargs=3, default=[0.485, 0.456, 0.406]) + parser.add_argument("--std", type=float, nargs=3, default=[0.229, 0.224, 0.225]) + parser.add_argument( + "--label-map", help="JSON mapping image folder names to class indices" + ) + parser.add_argument("--qat-epochs", type=float, default=0.0) + parser.add_argument("--qat-lr", type=float, default=2e-4) + parser.add_argument("--qat-batch", type=int, default=16) + parser.add_argument( + "--no-cpu-fallback", + action="store_true", + help="executorch mode: fail if any operator would run on the CPU", + ) + parser.add_argument( + "--device-min-sqnr", + type=float, + default=30.0, + help="on-board pass threshold (dB vs host) for programs with CPU operators", + ) + parser.add_argument( + "--crc-outputs", + action="store_true", + help="store CRC32 instead of full expected outputs (large outputs)", + ) + parser.add_argument( + "--min-sqnr", + type=float, + default=-math.inf, + help="fail when the device model's SQNR against float is lower", + ) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + + torch.set_grad_enabled(False) + torch.manual_seed(args.seed) + started = time.time() + shapes = [ + tuple(int(v) for v in s.split(",")) + for s in (args.input_shape or ["1,3,224,224"]) + ] + print(f"[1/5] Loading {args.model}") + model = remove_dropout( + fold_batch_norm(load_model(args.model, args.weights).eval())[0] + ).eval() + calibration, evaluation = make_data(args, shapes) + quantized, program = quantize(model, calibration, args) + compile_fn = compile_standalone if args.mode == "standalone" else compile_executorch + magic, blob, vectors, report, worst_float, worst_pt2e, totals = compile_fn( + args, model, quantized, program, evaluation + ) + print("[5/5] Writing artifacts") + name = args.name or Path(args.model.split(":")[-1]).stem + size = write_bundle(args.output, magic, [(name, blob, vectors)]) + report.update( + { + "name": name, + "model": args.model, + "bundle": str(args.output), + "bundle_bytes": size, + "calibration_samples": len(calibration), + "qat_epochs": args.qat_epochs, + "sqnr_vs_float_db": worst_float, + "sqnr_vs_pt2e_db": worst_pt2e, + "evaluation": totals, + } + ) + args.output.with_suffix(".json").write_text(json.dumps([report], indent=2)) + print( + f" bundle {args.output} ({size} bytes); report {args.output.with_suffix('.json')}" + ) + print( + f" embed it: CONFIG_ESPNN_RUNNER_BUNDLE (espnn_runner) or CONFIG_CAMERA_STREAM_MODEL (camera_stream) = {args.output.resolve()}" + ) + print(f" elapsed {time.time() - started:.1f} s") + if worst_float < args.min_sqnr: + print( + f"ESPNN COMPILE: FAIL (SQNR {worst_float:.2f} dB below {args.min_sqnr:.2f} dB)" + ) + return 1 + print("ESPNN COMPILE: PASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/espressif/benchmarks/esp32.json b/examples/espressif/benchmarks/esp32.json new file mode 100644 index 00000000000..e3d083d2593 --- /dev/null +++ b/examples/espressif/benchmarks/esp32.json @@ -0,0 +1,186 @@ +{ + "target": "esp32", + "chip": { + "target": "esp32", + "cores": "2", + "revision": "100", + "cpu_mhz": "240", + "flash_mb": "4", + "psram_kb": "0", + "internal_free_kb": "280", + "idf": "v5.5.5" + }, + "esptool": "ESP32-D0WDQ6 (revision v1.0)", + "date": "2026-10-18", + "iterations": 10, + "rows": [ + { + "name": "ds_cnn_kws", + "description": "Keyword spotting DS-CNN, 49x10 MFCC", + "input": "1,1,49,10", + "path": "standalone XTS3", + "operators": "12 int8", + "size": "0.03 MB", + "accuracy": "top-1 agrees with float 32/32", + "latency": "203.4 ms (203.4)", + "fps": "4.92", + "arena": "16 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "tiny_resnet_64", + "description": "Small residual CNN, 64x64", + "input": "1,3,64,64", + "path": "standalone XTS3", + "operators": "16 int8", + "size": "0.24 MB", + "accuracy": "top-1 agrees with float 32/32", + "latency": "705.0 ms (705.0)", + "fps": "1.42", + "arena": "36 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v3_small_96", + "description": "MobileNetV3-Small, ImageNet, 96x96", + "input": "1,3,96,96", + "path": "standalone XTS3", + "operators": "108 int8", + "size": "2.68 MB", + "accuracy": "top-1 0/32 (float 6), agrees with float 0/32", + "latency": "1440.7 ms (1440.7)", + "fps": "0.69", + "arena": "72 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v3_small_224", + "description": "MobileNetV3-Small, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "108 int8", + "size": "2.68 MB", + "accuracy": "top-1 6/32 (float 21), agrees with float 4/32", + "latency": "-", + "fps": "-", + "arena": "392 KB", + "check": "does not fit RAM (280 KB free, no PSRAM)" + }, + { + "name": "mobilenet_v2_224", + "description": "MobileNetV2, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "65 int8", + "size": "3.70 MB", + "accuracy": "top-1 20/32 (float 20), agrees with float 29/32", + "latency": "-", + "fps": "-", + "arena": "-", + "check": "does not fit 4 MB flash" + }, + { + "name": "mobilenet_v3_large_224", + "description": "MobileNetV3-Large, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "121 int8", + "size": "5.71 MB", + "accuracy": "top-1 24/32 (float 23), agrees with float 27/32", + "latency": "-", + "fps": "-", + "arena": "-", + "check": "does not fit 4 MB flash" + }, + { + "name": "efficientnet_b0_224", + "description": "EfficientNet-B0, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "190 int8", + "size": "5.65 MB", + "accuracy": "top-1 19/32 (float 23), agrees with float 23/32", + "latency": "-", + "fps": "-", + "arena": "-", + "check": "does not fit 4 MB flash" + } + ], + "raw_cases": { + "ds_cnn_kws": { + "name": "ds_cnn_kws", + "ops": "12", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "203431", + "min_us": "203419", + "max_us": "203490", + "iterations": "10", + "arena": "16000", + "arena_internal": "1", + "constants_in_ram": "0", + "blob": "31024", + "status": "PASS", + "kind": "standalone" + }, + "tiny_resnet_64": { + "name": "tiny_resnet_64", + "ops": "16", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "705045", + "min_us": "705029", + "max_us": "705066", + "iterations": "10", + "arena": "36864", + "arena_internal": "1", + "constants_in_ram": "0", + "blob": "242808", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v3_small_96": { + "name": "mobilenet_v3_small_96", + "ops": "108", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "1440734", + "min_us": "1440709", + "max_us": "1440760", + "iterations": "10", + "arena": "73728", + "arena_internal": "1", + "constants_in_ram": "0", + "blob": "2678784", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v3_small_224": { + "name": "mobilenet_v3_small_224", + "status": "FAIL", + "reason": "load", + "kind": "standalone", + "error": "unable to allocate 401408-byte activation arena" + } + }, + "failures": { + "mobilenet_v2_224": "does not fit 4 MB flash", + "efficientnet_b0_224": "does not fit 4 MB flash", + "mobilenet_v3_large_224": "does not fit 4 MB flash" + } +} \ No newline at end of file diff --git a/examples/espressif/benchmarks/esp32.md b/examples/espressif/benchmarks/esp32.md new file mode 100644 index 00000000000..913fb7f5311 --- /dev/null +++ b/examples/espressif/benchmarks/esp32.md @@ -0,0 +1,17 @@ +# ESP-NN backend benchmarks: esp32 + +Board: esp32, 2 cores at 240 MHz, 4 MB flash, 0 KB PSRAM, ESP-IDF v5.5.5. + +| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy | +|---|---|---|---|---|---|---|---|---|---| +| Keyword spotting DS-CNN, 49x10 MFCC | 1,1,49,10 | standalone XTS3 | 12 int8 | 0.03 MB | 16 KB | 203.4 ms (203.4) | 4.92 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | +| Small residual CNN, 64x64 | 1,3,64,64 | standalone XTS3 | 16 int8 | 0.24 MB | 36 KB | 705.0 ms (705.0) | 1.42 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | +| MobileNetV3-Small, ImageNet, 96x96 | 1,3,96,96 | standalone XTS3 | 108 int8 | 2.68 MB | 72 KB | 1440.7 ms (1440.7) | 0.69 | bit-exact 2/2, ANSI too | top-1 0/32 (float 6), agrees with float 0/32 | +| MobileNetV3-Small, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 108 int8 | 2.68 MB | 392 KB | - | - | does not fit RAM (280 KB free, no PSRAM) | top-1 6/32 (float 21), agrees with float 4/32 | +| MobileNetV2, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 65 int8 | 3.70 MB | - | - | - | does not fit 4 MB flash | top-1 20/32 (float 20), agrees with float 29/32 | +| MobileNetV3-Large, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 121 int8 | 5.71 MB | - | - | - | does not fit 4 MB flash | top-1 24/32 (float 23), agrees with float 27/32 | +| EfficientNet-B0, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 190 int8 | 5.65 MB | - | - | - | does not fit 4 MB flash | top-1 19/32 (float 23), agrees with float 23/32 | + +Latency is the model invocation only (no input preprocessing), timed over repeated runs after a bit-exact (or SQNR) check against the host reference. + +Host accuracy is the int8 model on the host (the board's output is bit-identical) after post-training quantization with 64 calibration images and no QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with random weights report top-1 agreement or SQNR against the float model instead. diff --git a/examples/espressif/benchmarks/esp32s3.json b/examples/espressif/benchmarks/esp32s3.json new file mode 100644 index 00000000000..bad7a12f31f --- /dev/null +++ b/examples/espressif/benchmarks/esp32s3.json @@ -0,0 +1,370 @@ +{ + "target": "esp32s3", + "chip": { + "target": "esp32s3", + "cores": "2", + "revision": "2", + "cpu_mhz": "240", + "flash_mb": "16", + "psram_kb": "8192", + "internal_free_kb": "252", + "idf": "v5.5.5" + }, + "esptool": "ESP32-S3 (QFN56) (revision v0.2)", + "date": "2026-10-18", + "iterations": 10, + "rows": [ + { + "name": "ds_cnn_kws", + "description": "Keyword spotting DS-CNN, 49x10 MFCC", + "input": "1,1,49,10", + "path": "standalone XTS3", + "operators": "12 int8", + "size": "0.03 MB", + "accuracy": "top-1 agrees with float 32/32", + "latency": "16.4 ms (16.4)", + "fps": "60.80", + "arena": "16 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "tiny_resnet_64", + "description": "Small residual CNN, 64x64", + "input": "1,3,64,64", + "path": "standalone XTS3", + "operators": "16 int8", + "size": "0.24 MB", + "accuracy": "top-1 agrees with float 32/32", + "latency": "40.3 ms (40.3)", + "fps": "24.80", + "arena": "36 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v3_small_96", + "description": "MobileNetV3-Small, ImageNet, 96x96", + "input": "1,3,96,96", + "path": "standalone XTS3", + "operators": "108 int8", + "size": "2.68 MB", + "accuracy": "top-1 0/32 (float 6), agrees with float 0/32", + "latency": "116.2 ms (116.2)", + "fps": "8.61", + "arena": "72 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v3_small_224", + "description": "MobileNetV3-Small, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "108 int8", + "size": "2.68 MB", + "accuracy": "top-1 6/32 (float 21), agrees with float 4/32", + "latency": "581.7 ms (581.7)", + "fps": "1.72", + "arena": "392 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v2_224", + "description": "MobileNetV2, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "65 int8", + "size": "3.70 MB", + "accuracy": "top-1 20/32 (float 20), agrees with float 29/32", + "latency": "1609.3 ms (1609.3)", + "fps": "0.62", + "arena": "1470 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "mobilenet_v3_large_224", + "description": "MobileNetV3-Large, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "121 int8", + "size": "5.71 MB", + "accuracy": "top-1 24/32 (float 23), agrees with float 27/32", + "latency": "1581.6 ms (1581.6)", + "fps": "0.63", + "arena": "980 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "efficientnet_b0_224", + "description": "EfficientNet-B0, ImageNet, 224x224", + "input": "1,3,224,224", + "path": "standalone XTS3", + "operators": "190 int8", + "size": "5.65 MB", + "accuracy": "top-1 19/32 (float 23), agrees with float 23/32", + "latency": "2498.3 ms (2498.3)", + "fps": "0.40", + "arena": "2352 KB", + "check": "bit-exact 2/2, ANSI too" + }, + { + "name": "lraspp_mobilenet_v3_seg", + "description": "LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224", + "input": "1,3,224,224", + "path": "ExecuTorch .pte", + "operators": "146 delegated, 0 CPU", + "size": "3.43 MB", + "accuracy": "6.1 dB vs float", + "latency": "3580.4 ms (3579.1); delegate 3580.5, CPU 0.0", + "fps": "0.28", + "arena": "1045 KB", + "check": "bit-exact" + }, + { + "name": "ssdlite320_mobilenet_v3_det", + "description": "SSDLite320 MobileNetV3 detection heads, COCO, 320x320", + "input": "1,3,320,320", + "path": "ExecuTorch .pte", + "operators": "243 delegated, 0 CPU", + "size": "3.72 MB", + "accuracy": "7.4 dB vs float", + "latency": "3830.3 ms (3830.0); delegate 3830.4, CPU 0.0", + "fps": "0.26", + "arena": "2000 KB", + "check": "bit-exact" + }, + { + "name": "tiny_transformer_fallback", + "description": "Small vision transformer: LayerNorm and attention on the CPU fallback", + "input": "1,3,64,64", + "path": "ExecuTorch .pte", + "operators": "11 delegated, 30 CPU", + "size": "0.04 MB", + "accuracy": "top-1 agrees with float 31/32", + "latency": "85.0 ms (84.9); delegate 22.1, CPU 62.8", + "fps": "11.77", + "arena": "38 KB", + "check": "39.3 dB vs host" + } + ], + "raw_cases": { + "ds_cnn_kws": { + "name": "ds_cnn_kws", + "ops": "12", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "16448", + "min_us": "16444", + "max_us": "16451", + "iterations": "10", + "arena": "16000", + "arena_internal": "1", + "constants_in_ram": "1", + "blob": "31024", + "status": "PASS", + "kind": "standalone" + }, + "tiny_resnet_64": { + "name": "tiny_resnet_64", + "ops": "16", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "40319", + "min_us": "40316", + "max_us": "40324", + "iterations": "10", + "arena": "36864", + "arena_internal": "1", + "constants_in_ram": "1", + "blob": "242808", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v3_small_96": { + "name": "mobilenet_v3_small_96", + "ops": "108", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "116161", + "min_us": "116159", + "max_us": "116166", + "iterations": "10", + "arena": "73728", + "arena_internal": "1", + "constants_in_ram": "1", + "blob": "2678784", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v3_small_224": { + "name": "mobilenet_v3_small_224", + "ops": "108", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "581747", + "min_us": "581736", + "max_us": "581804", + "iterations": "10", + "arena": "401408", + "arena_internal": "0", + "constants_in_ram": "1", + "blob": "2678784", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v2_224": { + "name": "mobilenet_v2_224", + "ops": "65", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "1609305", + "min_us": "1609302", + "max_us": "1609334", + "iterations": "10", + "arena": "1505280", + "arena_internal": "0", + "constants_in_ram": "1", + "blob": "3696944", + "status": "PASS", + "kind": "standalone" + }, + "efficientnet_b0_224": { + "name": "efficientnet_b0_224", + "ops": "190", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "2498306", + "min_us": "2498284", + "max_us": "2498400", + "iterations": "10", + "arena": "2408448", + "arena_internal": "0", + "constants_in_ram": "0", + "blob": "5652624", + "status": "PASS", + "kind": "standalone" + }, + "mobilenet_v3_large_224": { + "name": "mobilenet_v3_large_224", + "ops": "121", + "inputs": "1", + "outputs": "1", + "samples": "2", + "exact": "2", + "mismatched": "0", + "worst": "0", + "ansi": "1", + "us": "1581600", + "min_us": "1581592", + "max_us": "1581606", + "iterations": "10", + "arena": "1003520", + "arena_internal": "0", + "constants_in_ram": "1", + "blob": "5709680", + "status": "PASS", + "kind": "standalone" + }, + "tiny_transformer_fallback": { + "name": "tiny_transformer_fallback", + "inputs": "1", + "outputs": "1", + "samples": "2", + "delegates": "5", + "delegate_calls": "5", + "us": "84957", + "min_us": "84865", + "max_us": "85108", + "iterations": "10", + "delegate_us": "22127", + "cpu_us": "62830", + "sqnr_db": "39.3", + "max_abs": "0.00264995", + "mismatched": "5", + "exact_required": "0", + "min_sqnr": "30.0", + "load_us": "7186", + "planned": "90112", + "arena": "38960", + "status": "PASS", + "kind": "executorch" + }, + "lraspp_mobilenet_v3_seg": { + "name": "lraspp_mobilenet_v3_seg", + "inputs": "1", + "outputs": "1", + "samples": "2", + "delegates": "1", + "delegate_calls": "1", + "us": "3580418", + "min_us": "3579074", + "max_us": "3584194", + "iterations": "10", + "delegate_us": "3580532", + "cpu_us": "-114", + "sqnr_db": "999.0", + "max_abs": "0", + "mismatched": "0", + "exact_required": "1", + "min_sqnr": "30.0", + "load_us": "47890", + "planned": "4816896", + "arena": "1070160", + "status": "PASS", + "kind": "executorch" + }, + "ssdlite320_mobilenet_v3_det": { + "name": "ssdlite320_mobilenet_v3_det", + "inputs": "1", + "outputs": "2", + "samples": "2", + "delegates": "1", + "delegate_calls": "1", + "us": "3830285", + "min_us": "3829961", + "max_us": "3832876", + "iterations": "10", + "delegate_us": "3830380", + "cpu_us": "-95", + "sqnr_db": "999.0", + "max_abs": "0", + "mismatched": "0", + "exact_required": "1", + "min_sqnr": "30.0", + "load_us": "80055", + "planned": "2457728", + "arena": "2048000", + "status": "PASS", + "kind": "executorch" + } + }, + "failures": {} +} \ No newline at end of file diff --git a/examples/espressif/benchmarks/esp32s3.md b/examples/espressif/benchmarks/esp32s3.md new file mode 100644 index 00000000000..bdf17b92a0f --- /dev/null +++ b/examples/espressif/benchmarks/esp32s3.md @@ -0,0 +1,20 @@ +# ESP-NN backend benchmarks: esp32s3 + +Board: esp32s3, 2 cores at 240 MHz, 16 MB flash, 8192 KB PSRAM, ESP-IDF v5.5.5. + +| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy | +|---|---|---|---|---|---|---|---|---|---| +| Keyword spotting DS-CNN, 49x10 MFCC | 1,1,49,10 | standalone XTS3 | 12 int8 | 0.03 MB | 16 KB | 16.4 ms (16.4) | 60.80 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | +| Small residual CNN, 64x64 | 1,3,64,64 | standalone XTS3 | 16 int8 | 0.24 MB | 36 KB | 40.3 ms (40.3) | 24.80 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | +| MobileNetV3-Small, ImageNet, 96x96 | 1,3,96,96 | standalone XTS3 | 108 int8 | 2.68 MB | 72 KB | 116.2 ms (116.2) | 8.61 | bit-exact 2/2, ANSI too | top-1 0/32 (float 6), agrees with float 0/32 | +| MobileNetV3-Small, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 108 int8 | 2.68 MB | 392 KB | 581.7 ms (581.7) | 1.72 | bit-exact 2/2, ANSI too | top-1 6/32 (float 21), agrees with float 4/32 | +| MobileNetV2, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 65 int8 | 3.70 MB | 1470 KB | 1609.3 ms (1609.3) | 0.62 | bit-exact 2/2, ANSI too | top-1 20/32 (float 20), agrees with float 29/32 | +| MobileNetV3-Large, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 121 int8 | 5.71 MB | 980 KB | 1581.6 ms (1581.6) | 0.63 | bit-exact 2/2, ANSI too | top-1 24/32 (float 23), agrees with float 27/32 | +| EfficientNet-B0, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 190 int8 | 5.65 MB | 2352 KB | 2498.3 ms (2498.3) | 0.40 | bit-exact 2/2, ANSI too | top-1 19/32 (float 23), agrees with float 23/32 | +| LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224 | 1,3,224,224 | ExecuTorch .pte | 146 delegated, 0 CPU | 3.43 MB | 1045 KB | 3580.4 ms (3579.1); delegate 3580.5, CPU 0.0 | 0.28 | bit-exact | 6.1 dB vs float | +| SSDLite320 MobileNetV3 detection heads, COCO, 320x320 | 1,3,320,320 | ExecuTorch .pte | 243 delegated, 0 CPU | 3.72 MB | 2000 KB | 3830.3 ms (3830.0); delegate 3830.4, CPU 0.0 | 0.26 | bit-exact | 7.4 dB vs float | +| Small vision transformer: LayerNorm and attention on the CPU fallback | 1,3,64,64 | ExecuTorch .pte | 11 delegated, 30 CPU | 0.04 MB | 38 KB | 85.0 ms (84.9); delegate 22.1, CPU 62.8 | 11.77 | 39.3 dB vs host | top-1 agrees with float 31/32 | + +Latency is the model invocation only (no input preprocessing), timed over repeated runs after a bit-exact (or SQNR) check against the host reference. + +Host accuracy is the int8 model on the host (the board's output is bit-identical) after post-training quantization with 64 calibration images and no QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with random weights report top-1 agreement or SQNR against the float model instead. diff --git a/examples/espressif/camera_stream/CMakeLists.txt b/examples/espressif/camera_stream/CMakeLists.txt new file mode 100644 index 00000000000..e5b44af9a7b --- /dev/null +++ b/examples/espressif/camera_stream/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +cmake_minimum_required(VERSION 3.16) + +get_filename_component( + EXECUTORCH_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE +) +set(EXTRA_COMPONENT_DIRS "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime") + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(camera_stream) diff --git a/examples/espressif/camera_stream/README.md b/examples/espressif/camera_stream/README.md new file mode 100644 index 00000000000..1b4fb9c0b6e --- /dev/null +++ b/examples/espressif/camera_stream/README.md @@ -0,0 +1,30 @@ +# Camera inference stream (ESP32-S3) + +Live ImageNet classification on an ESP32-S3 camera board with the standalone +`espnn_runtime`, streamed over USB to a terminal or a browser. + +Tested with an OV3660 on the ESP32-S3-EYE / Freenove ESP32-S3-WROOM CAM +pinout (selectable in menuconfig, as is the XIAO ESP32S3 Sense). + +```bash +# 1. Compile a model (QAT recovers MobileNetV3-Small accuracy; plain PTQ loses most of it) +python ../aot_espnn_compiler.py --model mobilenet_v3_small --mode standalone \ + --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val \ + --calib-samples 1024 --qat-epochs 3 --qat-lr 1e-4 --output models/mobilenet_v3_small_224.xtb + +# 2. Build and flash (activated ESP-IDF shell) +./flash.sh /dev/ttyACM0 + +# 3. Watch: terminal dashboard, or Chrome at http://127.0.0.1:8765/ +python viewer.py +python viewer.py --web +``` + +The board captures JPEG frames from the sensor, decodes and center-crops them, +runs the model on core 1 and streams results, the model's input image and a +15 fps preview on core 0. On the ESP32-S3 with MobileNetV3-Small this gives +about 1.4 results per second with roughly 0.75 s from capture to result. + +Notes from bring-up: capture JPEG rather than RGB565 (continuous RGB565 +capture competes with inference for PSRAM bandwidth), keep the JPEG frame +buffers in internal RAM, and do not use the camera driver's PSRAM DMA mode. diff --git a/examples/espressif/camera_stream/flash.sh b/examples/espressif/camera_stream/flash.sh new file mode 100755 index 00000000000..d813bb1f2c4 --- /dev/null +++ b/examples/espressif/camera_stream/flash.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Build and flash the camera example from an activated ESP-IDF shell. A running +# viewer.py releases the serial port while this script flashes. +set -euo pipefail +PORT=${1:-${ESPPORT:-/dev/ttyACM0}} +cd "$(dirname "$0")" +mkdir -p build +idf.py build +touch build/.viewer_pause +trap 'rm -f build/.viewer_pause' EXIT +sleep 1 +idf.py -p "$PORT" -b 921600 flash diff --git a/examples/espressif/camera_stream/imagenet_classes.txt b/examples/espressif/camera_stream/imagenet_classes.txt new file mode 100644 index 00000000000..8bdc20a0864 --- /dev/null +++ b/examples/espressif/camera_stream/imagenet_classes.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane bird +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot tank suit +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/examples/espressif/camera_stream/main/CMakeLists.txt b/examples/espressif/camera_stream/main/CMakeLists.txt new file mode 100644 index 00000000000..67d0802ead0 --- /dev/null +++ b/examples/espressif/camera_stream/main/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +idf_component_register( + SRCS + "main.cpp" + "camera_stream.cpp" + "model_data.S" + INCLUDE_DIRS + "." + REQUIRES + espnn_runtime + esp32-camera + esp_jpeg + esp_timer + esp_rom + esp_driver_usb_serial_jtag +) + +idf_build_get_property(project_dir PROJECT_DIR) +if(IS_ABSOLUTE "${CONFIG_CAMERA_STREAM_MODEL}") + set(model "${CONFIG_CAMERA_STREAM_MODEL}") +else() + set(model "${project_dir}/${CONFIG_CAMERA_STREAM_MODEL}") +endif() +if(NOT EXISTS "${model}") + message( + FATAL_ERROR + "Camera model ${model} not found. Compile one with " + "examples/espressif/aot_espnn_compiler.py --mode standalone and set " + "CONFIG_CAMERA_STREAM_MODEL." + ) +endif() +set_source_files_properties( + "model_data.S" PROPERTIES COMPILE_DEFINITIONS "CAMERA_MODEL_FILE=\"${model}\"" + OBJECT_DEPENDS "${model}" +) diff --git a/examples/espressif/camera_stream/main/Kconfig.projbuild b/examples/espressif/camera_stream/main/Kconfig.projbuild new file mode 100644 index 00000000000..7164391251d --- /dev/null +++ b/examples/espressif/camera_stream/main/Kconfig.projbuild @@ -0,0 +1,77 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +menu "Camera inference stream" + + config CAMERA_STREAM_MODEL + string "Model to embed (XTS3 graph or XTB1 bundle; absolute or relative to this project)" + default "models/mobilenet_v3_small_224.xtb" + + + choice CAMERA_STREAM_PINOUT + prompt "Camera module pinout" + default CAMERA_STREAM_PINOUT_S3_EYE + + config CAMERA_STREAM_PINOUT_S3_EYE + bool "ESP32-S3-EYE, Freenove ESP32-S3-WROOM CAM, ESP32-S3-CAM N16R8" + + config CAMERA_STREAM_PINOUT_XIAO_SENSE + bool "Seeed XIAO ESP32S3 Sense" + endchoice + + config CAMERA_STREAM_SENSOR_JPEG + bool "Capture JPEG from the sensor (decode on the ESP32 for inference)" + default y + help + JPEG frames are about 15 times smaller than RGB565, so the camera + DMA does not compete with inference for PSRAM bandwidth and the + preview needs no software encoder. Turn off to capture RGB565. + + config CAMERA_STREAM_SENSOR_JPEG_QUALITY + int "Sensor JPEG quality (0 to 63, lower is better)" + range 4 63 + default 12 + depends on CAMERA_STREAM_SENSOR_JPEG + + config CAMERA_STREAM_DUAL_CORE + bool "Split ESP-NN kernels across both cores" + default n + help + Faster inference, but the ESP-NN worker on core 0 competes with + the JPEG preview task, so the preview frame rate drops. + + config CAMERA_STREAM_XCLK_MHZ + int "Camera XCLK frequency in MHz" + range 6 24 + default 20 + + config CAMERA_STREAM_PREVIEW + bool "Stream JPEG preview frames alongside the results" + default y + + config CAMERA_STREAM_PREVIEW_QUALITY + int "Preview JPEG quality (1 to 100, higher is better)" + range 1 100 + default 50 + depends on CAMERA_STREAM_PREVIEW + + config CAMERA_STREAM_PREVIEW_MAX_FPS + int "Preview frame rate limit" + range 1 60 + default 15 + depends on CAMERA_STREAM_PREVIEW + + config CAMERA_STREAM_VFLIP + bool "Flip the image vertically" + default n + + config CAMERA_STREAM_HMIRROR + bool "Mirror the image horizontally" + default n + +endmenu diff --git a/examples/espressif/camera_stream/main/camera_stream.cpp b/examples/espressif/camera_stream/main/camera_stream.cpp new file mode 100644 index 00000000000..7aa9640e219 --- /dev/null +++ b/examples/espressif/camera_stream/main/camera_stream.cpp @@ -0,0 +1,994 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +extern const uint8_t camera_model_start[]; +extern const uint8_t camera_model_end[]; +} + +namespace executorch { +namespace examples { +namespace espressif { + +using ::executorch::backends::espressif::Int8Interpreter; +using ::executorch::backends::espressif::Int8RuntimeOptions; +using ::executorch::backends::espressif::Int8TensorInfo; +using ::executorch::backends::espressif::read_le_u32; + +namespace { + +constexpr int TOP_K = 5; +constexpr int CROP_NUMERATOR = 224; +constexpr int CROP_DENOMINATOR = 256; +constexpr float IMAGENET_MEAN[3] = {0.485f, 0.456f, 0.406f}; +constexpr float IMAGENET_STD[3] = {0.229f, 0.224f, 0.225f}; +constexpr size_t USB_TX_BUFFER_BYTES = 8 * 1024; +constexpr size_t SENSOR_JPEG_BUFFER_BYTES = 32 * 1024; +constexpr TickType_t USB_WRITE_TIMEOUT = pdMS_TO_TICKS(200); +constexpr size_t RESULT_QUEUE_LENGTH = 4; + +#if CONFIG_CAMERA_STREAM_VFLIP +constexpr int VFLIP = 1; +#else +constexpr int VFLIP = 0; +#endif +#if CONFIG_CAMERA_STREAM_HMIRROR +constexpr int HMIRROR = 1; +#else +constexpr int HMIRROR = 0; +#endif +#if CONFIG_CAMERA_STREAM_PREVIEW +constexpr bool PREVIEW = true; +constexpr int PREVIEW_QUALITY = CONFIG_CAMERA_STREAM_PREVIEW_QUALITY; +constexpr int PREVIEW_MAX_FPS = CONFIG_CAMERA_STREAM_PREVIEW_MAX_FPS; +#else +constexpr bool PREVIEW = false; +constexpr int PREVIEW_QUALITY = 50; +constexpr int PREVIEW_MAX_FPS = 1; +#endif +#if CONFIG_CAMERA_STREAM_SENSOR_JPEG +constexpr bool SENSOR_JPEG = true; +constexpr int SENSOR_JPEG_QUALITY = CONFIG_CAMERA_STREAM_SENSOR_JPEG_QUALITY; +#else +constexpr bool SENSOR_JPEG = false; +constexpr int SENSOR_JPEG_QUALITY = 12; +#endif + +struct Pinout { + const char* name; + int pwdn, reset, xclk, sda, scl; + int d7, d6, d5, d4, d3, d2, d1, d0; + int vsync, href, pclk; +}; + +#if CONFIG_CAMERA_STREAM_PINOUT_XIAO_SENSE +constexpr Pinout PINOUT = { + "XIAO ESP32S3 Sense", + -1, + -1, + 10, + 40, + 39, + 48, + 11, + 12, + 14, + 16, + 18, + 17, + 15, + 38, + 47, + 13}; +#else +constexpr Pinout PINOUT = { + "ESP32-S3-EYE/Freenove", + -1, + -1, + 15, + 4, + 5, + 16, + 17, + 18, + 12, + 10, + 8, + 9, + 11, + 6, + 7, + 13}; +#endif + +struct Prediction { + int32_t index; + float probability; +}; + +struct Result { + uint32_t seq; + int64_t decode_us; + int64_t preprocess_us; + int64_t invoke_us; + int64_t cycle_us; + int64_t latency_us; + uint32_t dropped_results; + uint32_t decode_errors; + Prediction top[TOP_K]; +}; + +struct Shared { + SemaphoreHandle_t state_lock; + QueueHandle_t results; + Int8Interpreter* interpreter; + int32_t input_width; + int32_t input_height; + bool decoder_swaps_red_blue; + uint8_t* published_bgr; + bool published_ready; + uint32_t published_seq; +}; + +Shared g_shared{}; + +void* psram_alloc(size_t bytes) { + void* pointer = heap_caps_malloc(bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (pointer == nullptr) + pointer = heap_caps_malloc(bytes, MALLOC_CAP_8BIT); + return pointer; +} + +int64_t frame_time_us(const camera_fb_t* frame) { + return static_cast(frame->timestamp.tv_sec) * 1000000LL + + frame->timestamp.tv_usec; +} + +bool decode_jpeg_rgb888( + const uint8_t* jpeg, + size_t length, + uint8_t* output, + size_t output_size, + int32_t& width, + int32_t& height) { + esp_jpeg_image_cfg_t config{}; + config.indata = const_cast(jpeg); + config.indata_size = static_cast(length); + config.outbuf = output; + config.outbuf_size = static_cast(output_size); + config.out_format = JPEG_IMAGE_FORMAT_RGB888; + config.out_scale = JPEG_IMAGE_SCALE_0; + esp_jpeg_image_output_t image{}; + if (esp_jpeg_decode(&config, &image) != ESP_OK) + return false; + width = image.width; + height = image.height; + return true; +} + +bool probe_decoder_channel_order(bool& swaps_red_blue) { + constexpr int SIDE = 16; + uint8_t bgr[SIDE * SIDE * 3]; + for (int i = 0; i < SIDE * SIDE; ++i) { + bgr[i * 3 + 0] = 0; + bgr[i * 3 + 1] = 0; + bgr[i * 3 + 2] = 255; + } + uint8_t* jpeg = nullptr; + size_t length = 0; + if (!fmt2jpg( + bgr, sizeof(bgr), SIDE, SIDE, PIXFORMAT_RGB888, 90, &jpeg, &length)) + return false; + uint8_t decoded[SIDE * SIDE * 3]; + int32_t width = 0; + int32_t height = 0; + const bool ok = + decode_jpeg_rgb888(jpeg, length, decoded, sizeof(decoded), width, height); + std::free(jpeg); + if (!ok || width != SIDE || height != SIDE) + return false; + const int center = (SIDE / 2 * SIDE + SIDE / 2) * 3; + std::printf( + "Decoder check: pure red decodes to bytes %u,%u,%u\n", + decoded[center], + decoded[center + 1], + decoded[center + 2]); + if (decoded[center] > 200 && decoded[center + 2] < 60) { + swaps_red_blue = false; + return true; + } + if (decoded[center + 2] > 200 && decoded[center] < 60) { + swaps_red_blue = true; + return true; + } + return false; +} + +constexpr char BASE64_ALPHABET[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +size_t base64_encode(const uint8_t* source, size_t length, char* destination) { + size_t o = 0; + size_t i = 0; + for (; i + 2 < length; i += 3) { + const uint32_t v = (static_cast(source[i]) << 16) | + (static_cast(source[i + 1]) << 8) | source[i + 2]; + destination[o++] = BASE64_ALPHABET[(v >> 18) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 12) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 6) & 63]; + destination[o++] = BASE64_ALPHABET[v & 63]; + } + if (i < length) { + uint32_t v = static_cast(source[i]) << 16; + if (i + 1 < length) + v |= static_cast(source[i + 1]) << 8; + destination[o++] = BASE64_ALPHABET[(v >> 18) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 12) & 63]; + destination[o++] = i + 1 < length ? BASE64_ALPHABET[(v >> 6) & 63] : '='; + destination[o++] = '='; + } + return o; +} + +class UsbLink { + public: + static bool begin() { + usb_serial_jtag_driver_config_t config = + USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(); + config.tx_buffer_size = USB_TX_BUFFER_BYTES; + config.rx_buffer_size = 256; + if (!usb_serial_jtag_is_driver_installed() && + usb_serial_jtag_driver_install(&config) != ESP_OK) + return false; + std::fflush(stdout); + usb_serial_jtag_vfs_use_driver(); + return true; + } + + bool write_line(const char* data, size_t length) { + size_t sent = 0; + while (sent < length) { + const int written = usb_serial_jtag_write_bytes( + data + sent, length - sent, USB_WRITE_TIMEOUT); + if (written <= 0) { + if (sent != 0) + usb_serial_jtag_write_bytes("\n", 1, 0); + host_reading_ = false; + ++failed_writes_; + return false; + } + sent += static_cast(written); + } + host_reading_ = true; + return true; + } + + bool probe() { + if (usb_serial_jtag_write_bytes("\n", 1, USB_WRITE_TIMEOUT) > 0) + host_reading_ = true; + return host_reading_; + } + + bool host_reading() const { + return host_reading_; + } + uint32_t failed_writes() const { + return failed_writes_; + } + + private: + bool host_reading_ = true; + uint32_t failed_writes_ = 0; +}; + +class JpegLine { + public: + bool reserve(size_t bytes) { + if (bytes <= capacity_) + return true; + std::free(buffer_); + buffer_ = static_cast(psram_alloc(bytes)); + capacity_ = buffer_ != nullptr ? bytes : 0; + return buffer_ != nullptr; + } + + size_t build( + const char* kind, + uint32_t seq, + uint32_t result_seq, + int32_t width, + int32_t height, + int64_t encode_us, + const uint8_t* jpeg, + size_t length) { + const size_t needed = 192 + ((length + 2) / 3) * 4; + if (!reserve(needed)) + return 0; + const uint32_t crc = esp_rom_crc32_le(0, jpeg, length); + const int header = std::snprintf( + buffer_, + capacity_, + "#JPG kind=%s seq=%u res=%u w=%ld h=%ld enc_us=%lld len=%u crc=%08lx b64=", + kind, + static_cast(seq), + static_cast(result_seq), + static_cast(width), + static_cast(height), + static_cast(encode_us), + static_cast(length), + static_cast(crc)); + if (header <= 0) + return 0; + size_t total = static_cast(header); + total += base64_encode(jpeg, length, buffer_ + total); + buffer_[total++] = '\n'; + return total; + } + + bool send(UsbLink& link, size_t length) const { + return length != 0 && link.write_line(buffer_, length); + } + + private: + char* buffer_ = nullptr; + size_t capacity_ = 0; +}; + +uint8_t expand5(uint32_t v) { + return static_cast((v << 3) | (v >> 2)); +} +uint8_t expand6(uint32_t v) { + return static_cast((v << 2) | (v >> 4)); +} + +class Resampler { + public: + bool init( + int32_t source_width, + int32_t source_height, + int32_t output_width, + int32_t output_height) { + source_width_ = source_width; + output_width_ = output_width; + output_height_ = output_height; + int32_t side = source_width < source_height ? source_width : source_height; + side = side * CROP_NUMERATOR / CROP_DENOMINATOR; + crop_x_ = (source_width - side) / 2; + crop_y_ = (source_height - side) / 2; + crop_side_ = side; + x_index_ = + static_cast(std::malloc(sizeof(int32_t) * output_width * 2)); + y_index_ = + static_cast(std::malloc(sizeof(int32_t) * output_height * 2)); + x_weight_ = + static_cast(std::malloc(sizeof(uint16_t) * output_width)); + y_weight_ = + static_cast(std::malloc(sizeof(uint16_t) * output_height)); + if (x_index_ == nullptr || y_index_ == nullptr || x_weight_ == nullptr || + y_weight_ == nullptr) + return false; + build(output_width, crop_x_, x_index_, x_weight_); + build(output_height, crop_y_, y_index_, y_weight_); + return true; + } + + void run_rgb565_be(const uint8_t* source, uint8_t* rgb888) const { + for (int32_t oy = 0; oy < output_height_; ++oy) { + const uint8_t* row0 = + source + static_cast(y_index_[oy * 2]) * source_width_ * 2; + const uint8_t* row1 = source + + static_cast(y_index_[oy * 2 + 1]) * source_width_ * 2; + const uint32_t wy = y_weight_[oy]; + uint8_t* out = rgb888 + static_cast(oy) * output_width_ * 3; + for (int32_t ox = 0; ox < output_width_; ++ox) { + const int32_t x0 = x_index_[ox * 2] * 2; + const int32_t x1 = x_index_[ox * 2 + 1] * 2; + const uint32_t wx = x_weight_[ox]; + const uint32_t p00 = + (static_cast(row0[x0]) << 8) | row0[x0 + 1]; + const uint32_t p01 = + (static_cast(row0[x1]) << 8) | row0[x1 + 1]; + const uint32_t p10 = + (static_cast(row1[x0]) << 8) | row1[x0 + 1]; + const uint32_t p11 = + (static_cast(row1[x1]) << 8) | row1[x1 + 1]; + out[ox * 3 + 0] = blend( + expand5(p00 >> 11), + expand5(p01 >> 11), + expand5(p10 >> 11), + expand5(p11 >> 11), + wx, + wy); + out[ox * 3 + 1] = blend( + expand6((p00 >> 5) & 63), + expand6((p01 >> 5) & 63), + expand6((p10 >> 5) & 63), + expand6((p11 >> 5) & 63), + wx, + wy); + out[ox * 3 + 2] = blend( + expand5(p00 & 31), + expand5(p01 & 31), + expand5(p10 & 31), + expand5(p11 & 31), + wx, + wy); + } + } + } + + void run_rgb888(const uint8_t* source, bool swap_red_blue, uint8_t* rgb888) + const { + const int red = swap_red_blue ? 2 : 0; + const int blue = swap_red_blue ? 0 : 2; + for (int32_t oy = 0; oy < output_height_; ++oy) { + const uint8_t* row0 = + source + static_cast(y_index_[oy * 2]) * source_width_ * 3; + const uint8_t* row1 = source + + static_cast(y_index_[oy * 2 + 1]) * source_width_ * 3; + const uint32_t wy = y_weight_[oy]; + uint8_t* out = rgb888 + static_cast(oy) * output_width_ * 3; + for (int32_t ox = 0; ox < output_width_; ++ox) { + const int32_t x0 = x_index_[ox * 2] * 3; + const int32_t x1 = x_index_[ox * 2 + 1] * 3; + const uint32_t wx = x_weight_[ox]; + out[ox * 3 + 0] = blend( + row0[x0 + red], + row0[x1 + red], + row1[x0 + red], + row1[x1 + red], + wx, + wy); + out[ox * 3 + 1] = blend( + row0[x0 + 1], row0[x1 + 1], row1[x0 + 1], row1[x1 + 1], wx, wy); + out[ox * 3 + 2] = blend( + row0[x0 + blue], + row0[x1 + blue], + row1[x0 + blue], + row1[x1 + blue], + wx, + wy); + } + } + } + + int32_t crop_x() const { + return crop_x_; + } + int32_t crop_y() const { + return crop_y_; + } + int32_t crop_side() const { + return crop_side_; + } + + private: + static uint8_t blend( + uint32_t a, + uint32_t b, + uint32_t c, + uint32_t d, + uint32_t wx, + uint32_t wy) { + const uint32_t top = a * (256 - wx) + b * wx; + const uint32_t bottom = c * (256 - wx) + d * wx; + return static_cast((top * (256 - wy) + bottom * wy + 32768) >> 16); + } + + void build(int32_t output, int32_t offset, int32_t* index, uint16_t* weight) + const { + const int32_t last = crop_side_ - 1; + for (int32_t o = 0; o < output; ++o) { + int32_t position = + static_cast( + (static_cast(2 * o + 1) * crop_side_ * 256) / + (2 * output)) - + 128; + if (position < 0) + position = 0; + int32_t i0 = position >> 8; + if (i0 > last) + i0 = last; + const int32_t i1 = i0 < last ? i0 + 1 : last; + index[o * 2] = offset + i0; + index[o * 2 + 1] = offset + i1; + weight[o] = static_cast(i0 < last ? position & 255 : 0); + } + } + + int32_t source_width_ = 0; + int32_t output_width_ = 0; + int32_t output_height_ = 0; + int32_t crop_x_ = 0; + int32_t crop_y_ = 0; + int32_t crop_side_ = 0; + int32_t* x_index_ = nullptr; + int32_t* y_index_ = nullptr; + uint16_t* x_weight_ = nullptr; + uint16_t* y_weight_ = nullptr; +}; + +bool top_k( + const Int8Interpreter& interpreter, + float* scratch, + size_t count, + Prediction* top) { + if (!interpreter.dequantize_output(0, scratch, count)) + return false; + float maximum = scratch[0]; + for (size_t i = 1; i < count; ++i) { + if (scratch[i] > maximum) + maximum = scratch[i]; + } + double sum = 0.0; + for (size_t i = 0; i < count; ++i) { + scratch[i] = std::exp(scratch[i] - maximum); + sum += scratch[i]; + } + for (int k = 0; k < TOP_K; ++k) { + top[k].index = -1; + top[k].probability = -1.0f; + } + for (size_t i = 0; i < count; ++i) { + const float p = static_cast(scratch[i] / sum); + if (p <= top[TOP_K - 1].probability) + continue; + int k = TOP_K - 1; + while (k > 0 && p > top[k - 1].probability) { + top[k] = top[k - 1]; + --k; + } + top[k].index = static_cast(i); + top[k].probability = p; + } + return true; +} + +void inference_task(void*) { + Int8Interpreter& interpreter = *g_shared.interpreter; + Int8TensorInfo output_info{}; + interpreter.output_info(0, output_info); + const int32_t width = g_shared.input_width; + const int32_t height = g_shared.input_height; + const size_t rgb_bytes = static_cast(width) * height * 3; + uint8_t* rgb = static_cast(psram_alloc(rgb_bytes)); + float* scratch = + static_cast(std::malloc(sizeof(float) * output_info.bytes)); + uint8_t* decoded = nullptr; + size_t decoded_size = 0; + Resampler resampler; + bool resampler_ready = false; + if (rgb == nullptr || scratch == nullptr) { + std::printf("#ERR inference buffers could not be allocated\n"); + vTaskDelete(nullptr); + return; + } + uint32_t seq = 0; + uint32_t dropped = 0; + uint32_t decode_errors = 0; + int64_t previous = esp_timer_get_time(); + while (true) { + camera_fb_t* frame = esp_camera_fb_get(); + if (frame == nullptr) + continue; + const int64_t started = esp_timer_get_time(); + const int64_t captured = frame_time_us(frame); + const int32_t frame_width = static_cast(frame->width); + const int32_t frame_height = static_cast(frame->height); + if (!resampler_ready) { + resampler_ready = + resampler.init(frame_width, frame_height, width, height); + std::printf( + "#CROP x=%ld y=%ld side=%ld\n", + static_cast(resampler.crop_x()), + static_cast(resampler.crop_y()), + static_cast(resampler.crop_side())); + } + int64_t decode_us = 0; + if (SENSOR_JPEG) { + const size_t needed = static_cast(frame_width) * frame_height * 3; + if (decoded_size < needed) { + std::free(decoded); + decoded = static_cast(psram_alloc(needed)); + decoded_size = decoded != nullptr ? needed : 0; + } + int32_t decoded_width = 0; + int32_t decoded_height = 0; + const bool ok = decoded != nullptr && + decode_jpeg_rgb888( + frame->buf, + frame->len, + decoded, + decoded_size, + decoded_width, + decoded_height) && + decoded_width == frame_width && decoded_height == frame_height; + esp_camera_fb_return(frame); + decode_us = esp_timer_get_time() - started; + if (!ok) { + ++decode_errors; + continue; + } + resampler.run_rgb888(decoded, g_shared.decoder_swaps_red_blue, rgb); + } else { + resampler.run_rgb565_be(frame->buf, rgb); + esp_camera_fb_return(frame); + } + interpreter.set_input_rgb888( + 0, rgb, width, height, IMAGENET_MEAN, IMAGENET_STD); + const int64_t preprocessed = esp_timer_get_time(); + if (!interpreter.invoke()) { + std::printf("#ERR invoke failed\n"); + continue; + } + Result result{}; + result.seq = ++seq; + result.decode_us = decode_us; + result.preprocess_us = preprocessed - started; + result.invoke_us = interpreter.last_invoke_us(); + top_k(interpreter, scratch, output_info.bytes, result.top); + const int64_t now = esp_timer_get_time(); + result.cycle_us = now - previous; + result.latency_us = now - captured; + result.dropped_results = dropped; + result.decode_errors = decode_errors; + previous = now; + if (xQueueSend(g_shared.results, &result, 0) != pdTRUE) + ++dropped; + if (PREVIEW) { + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + uint8_t* destination = g_shared.published_bgr; + for (size_t i = 0; i < rgb_bytes; i += 3) { + destination[i] = rgb[i + 2]; + destination[i + 1] = rgb[i + 1]; + destination[i + 2] = rgb[i]; + } + g_shared.published_ready = true; + g_shared.published_seq = result.seq; + xSemaphoreGive(g_shared.state_lock); + } + vTaskDelay(1); + } +} + +void send_result(UsbLink& link, const Result& result) { + char line[352]; + int n = std::snprintf( + line, + sizeof(line), + "#RES seq=%u dec_us=%lld pre_us=%lld infer_us=%lld cycle_us=%lld latency_us=%lld " + "dropped=%u dec_err=%u usb_fail=%u top=", + static_cast(result.seq), + static_cast(result.decode_us), + static_cast(result.preprocess_us), + static_cast(result.invoke_us), + static_cast(result.cycle_us), + static_cast(result.latency_us), + static_cast(result.dropped_results), + static_cast(result.decode_errors), + static_cast(link.failed_writes())); + for (int k = 0; k < TOP_K && n > 0 && n < static_cast(sizeof(line)) - 24; + ++k) { + n += std::snprintf( + line + n, + sizeof(line) - n, + "%s%ld:%.4f", + k ? "," : "", + static_cast(result.top[k].index), + static_cast(result.top[k].probability)); + } + if (n <= 0 || n >= static_cast(sizeof(line)) - 1) + return; + line[n++] = '\n'; + link.write_line(line, static_cast(n)); +} + +void send_model_input( + UsbLink& link, + JpegLine& line, + uint8_t* input_copy, + size_t rgb_bytes) { + bool have_input = false; + uint32_t input_seq = 0; + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + if (g_shared.published_ready) { + std::memcpy(input_copy, g_shared.published_bgr, rgb_bytes); + g_shared.published_ready = false; + input_seq = g_shared.published_seq; + have_input = true; + } + xSemaphoreGive(g_shared.state_lock); + if (!have_input || !link.host_reading()) + return; + uint8_t* jpeg = nullptr; + size_t length = 0; + const int64_t started = esp_timer_get_time(); + if (fmt2jpg( + input_copy, + rgb_bytes, + static_cast(g_shared.input_width), + static_cast(g_shared.input_height), + PIXFORMAT_RGB888, + PREVIEW_QUALITY, + &jpeg, + &length)) { + line.send( + link, + line.build( + "input", + input_seq, + input_seq, + g_shared.input_width, + g_shared.input_height, + esp_timer_get_time() - started, + jpeg, + length)); + } + std::free(jpeg); +} + +void send_preview_frame(UsbLink& link, JpegLine& line, uint32_t seq) { + camera_fb_t* frame = esp_camera_fb_get(); + if (frame == nullptr) + return; + const int32_t frame_width = static_cast(frame->width); + const int32_t frame_height = static_cast(frame->height); + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + const uint32_t result_seq = g_shared.published_seq; + xSemaphoreGive(g_shared.state_lock); + size_t length = 0; + if (SENSOR_JPEG) { + length = line.build( + "cam", + seq, + result_seq, + frame_width, + frame_height, + 0, + frame->buf, + frame->len); + esp_camera_fb_return(frame); + } else { + uint8_t* jpeg = nullptr; + size_t jpeg_length = 0; + const int64_t started = esp_timer_get_time(); + const bool encoded = frame2jpg(frame, PREVIEW_QUALITY, &jpeg, &jpeg_length); + const int64_t encode_us = esp_timer_get_time() - started; + esp_camera_fb_return(frame); + if (encoded) { + length = line.build( + "cam", + seq, + result_seq, + frame_width, + frame_height, + encode_us, + jpeg, + jpeg_length); + } + std::free(jpeg); + } + line.send(link, length); +} + +void stream_task(void*) { + UsbLink link; + if (!link.begin()) { + std::printf("#ERR USB-Serial-JTAG driver install failed\n"); + vTaskDelete(nullptr); + return; + } + JpegLine line; + const size_t rgb_bytes = + static_cast(g_shared.input_width) * g_shared.input_height * 3; + uint8_t* input_copy = + PREVIEW ? static_cast(psram_alloc(rgb_bytes)) : nullptr; + if (PREVIEW && (input_copy == nullptr || !line.reserve(64 * 1024))) { + std::printf("#ERR preview buffers could not be allocated\n"); + vTaskDelete(nullptr); + return; + } + const int64_t period_us = 1000000LL / PREVIEW_MAX_FPS; + int64_t next_preview = esp_timer_get_time(); + uint32_t preview_seq = 0; + while (true) { + TickType_t wait = portMAX_DELAY; + if (PREVIEW) { + const int64_t remaining = next_preview - esp_timer_get_time(); + wait = remaining > 0 ? pdMS_TO_TICKS(remaining / 1000) : 0; + } + Result result; + if (xQueueReceive(g_shared.results, &result, wait) == pdTRUE) { + send_result(link, result); + if (PREVIEW) + send_model_input(link, line, input_copy, rgb_bytes); + continue; + } + if (!PREVIEW) + continue; + const int64_t now = esp_timer_get_time(); + next_preview = now - next_preview > period_us ? now + period_us + : next_preview + period_us; + if (!link.host_reading()) { + next_preview = now + 500000; + link.probe(); + continue; + } + send_preview_frame(link, line, ++preview_seq); + } +} + +bool start_camera() { + camera_config_t config{}; + config.pin_pwdn = PINOUT.pwdn; + config.pin_reset = PINOUT.reset; + config.pin_xclk = PINOUT.xclk; + config.pin_sccb_sda = PINOUT.sda; + config.pin_sccb_scl = PINOUT.scl; + config.pin_d7 = PINOUT.d7; + config.pin_d6 = PINOUT.d6; + config.pin_d5 = PINOUT.d5; + config.pin_d4 = PINOUT.d4; + config.pin_d3 = PINOUT.d3; + config.pin_d2 = PINOUT.d2; + config.pin_d1 = PINOUT.d1; + config.pin_d0 = PINOUT.d0; + config.pin_vsync = PINOUT.vsync; + config.pin_href = PINOUT.href; + config.pin_pclk = PINOUT.pclk; + config.xclk_freq_hz = CONFIG_CAMERA_STREAM_XCLK_MHZ * 1000000; + config.ledc_timer = LEDC_TIMER_0; + config.ledc_channel = LEDC_CHANNEL_0; + config.pixel_format = SENSOR_JPEG ? PIXFORMAT_JPEG : PIXFORMAT_RGB565; + config.frame_size = FRAMESIZE_QVGA; + config.jpeg_quality = SENSOR_JPEG_QUALITY; + config.fb_count = SENSOR_JPEG ? 2 : 3; + config.fb_location = SENSOR_JPEG ? CAMERA_FB_IN_DRAM : CAMERA_FB_IN_PSRAM; + config.grab_mode = CAMERA_GRAB_LATEST; + config.sccb_i2c_port = 0; + config.jpeg_buffer_size = SENSOR_JPEG ? SENSOR_JPEG_BUFFER_BYTES : 0; + const esp_err_t error = esp_camera_init(&config); + if (error != ESP_OK) { + std::printf( + "#ERR camera init failed with pinout %s: 0x%x\n", + PINOUT.name, + static_cast(error)); + return false; + } + sensor_t* sensor = esp_camera_sensor_get(); + const camera_sensor_info_t* info = + sensor ? esp_camera_sensor_get_info(&sensor->id) : nullptr; + if (sensor != nullptr) { + sensor->set_vflip(sensor, VFLIP); + sensor->set_hmirror(sensor, HMIRROR); + } + std::printf( + "Camera: %s (PID 0x%04x) on pinout %s, QVGA %s, XCLK %d MHz\n", + info ? info->name : "unknown", + sensor ? static_cast(sensor->id.PID) : 0U, + PINOUT.name, + SENSOR_JPEG ? "JPEG" : "RGB565", + CONFIG_CAMERA_STREAM_XCLK_MHZ); + return true; +} + +} // namespace + +void run_camera_stream() { + std::printf("\n========================================\n"); + std::printf(" CAMERA INFERENCE STREAM\n"); + std::printf("========================================\n"); + if (SENSOR_JPEG && + !probe_decoder_channel_order(g_shared.decoder_swaps_red_blue)) { + std::printf("#ERR JPEG decoder channel order check failed\n"); + return; + } + if (!start_camera()) + return; + + Int8RuntimeOptions options{}; + options.copy_constants_to_psram = true; +#if CONFIG_CAMERA_STREAM_DUAL_CORE + options.dual_core = true; +#endif + Int8Interpreter* interpreter = new (std::nothrow) Int8Interpreter(); + const uint8_t* model = camera_model_start; + size_t model_size = reinterpret_cast(camera_model_end) - + reinterpret_cast(camera_model_start); + if (model_size >= 12 + 48 && std::memcmp(model, "XTB1", 4) == 0) { + const uint32_t offset = read_le_u32(model + 12 + 32); + const uint32_t length = read_le_u32(model + 12 + 36); + if (offset > model_size || length > model_size - offset) { + std::printf("#ERR model bundle is truncated\n"); + return; + } + model += offset; + model_size = length; + } + if (interpreter == nullptr || + !interpreter->load(model, model_size, options)) { + std::printf("#ERR model load failed\n"); + return; + } + Int8TensorInfo input{}; + Int8TensorInfo output{}; + if (interpreter->input_count() != 1 || !interpreter->input_info(0, input) || + !interpreter->output_info(0, output) || input.channels != 3) { + std::printf("#ERR model must have one 3-channel image input\n"); + return; + } + g_shared.interpreter = interpreter; + g_shared.input_width = input.width; + g_shared.input_height = input.height; + g_shared.state_lock = xSemaphoreCreateMutex(); + g_shared.results = xQueueCreate(RESULT_QUEUE_LENGTH, sizeof(Result)); + g_shared.published_bgr = PREVIEW + ? static_cast( + psram_alloc(static_cast(input.width) * input.height * 3)) + : nullptr; + if (g_shared.state_lock == nullptr || g_shared.results == nullptr || + (PREVIEW && g_shared.published_bgr == nullptr)) { + std::printf("#ERR stream state could not be allocated\n"); + return; + } + std::printf( + "#CAM model_input=%ldx%ld classes=%u dual_core=%d preview=%d preview_fps=%d " + "sensor=%s sensor_quality=%d xclk_mhz=%d constants_in_psram=%d\n", + static_cast(input.width), + static_cast(input.height), + static_cast(output.bytes), + options.dual_core ? 1 : 0, + PREVIEW ? 1 : 0, + PREVIEW_MAX_FPS, + SENSOR_JPEG ? "jpeg" : "rgb565", + SENSOR_JPEG_QUALITY, + CONFIG_CAMERA_STREAM_XCLK_MHZ, + interpreter->constants_copied() ? 1 : 0); + std::printf( + "Internal RAM free %u (largest %u), PSRAM free %u\n", + static_cast( + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_largest_free_block( + MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM))); + xTaskCreatePinnedToCore( + stream_task, "cam_stream", 8 * 1024, nullptr, 4, nullptr, 0); + xTaskCreatePinnedToCore( + inference_task, "cam_infer", 16 * 1024, nullptr, 5, nullptr, 1); +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/camera_stream.h b/examples/espressif/camera_stream/main/camera_stream.h new file mode 100644 index 00000000000..2d2c205e297 --- /dev/null +++ b/examples/espressif/camera_stream/main/camera_stream.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +namespace executorch { +namespace examples { +namespace espressif { + +/** + * Loads the embedded model, starts the camera and creates the inference + * (core 1) and USB streaming (core 0) tasks, then returns. On a load or + * camera failure it prints the reason and returns without starting tasks. + */ +void run_camera_stream(); + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/idf_component.yml b/examples/espressif/camera_stream/main/idf_component.yml new file mode 100644 index 00000000000..18dc3be57f4 --- /dev/null +++ b/examples/espressif/camera_stream/main/idf_component.yml @@ -0,0 +1,12 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +dependencies: + idf: + version: ">=5.1" + espressif/esp32-camera: "^2.1.8" diff --git a/examples/espressif/camera_stream/main/main.cpp b/examples/espressif/camera_stream/main/main.cpp new file mode 100644 index 00000000000..6b4e8769b48 --- /dev/null +++ b/examples/espressif/camera_stream/main/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include + +namespace { + +void camera_task(void*) { + executorch::examples::espressif::run_camera_stream(); + vTaskDelete(nullptr); +} + +} // namespace + +extern "C" void app_main() { + xTaskCreatePinnedToCore( + camera_task, "camera_setup", 16 * 1024, nullptr, 5, nullptr, 0); +} diff --git a/examples/espressif/camera_stream/main/model_data.S b/examples/espressif/camera_stream/main/model_data.S new file mode 100644 index 00000000000..564dfdf2f8e --- /dev/null +++ b/examples/espressif/camera_stream/main/model_data.S @@ -0,0 +1,20 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + .section .rodata.camera_model, "a" + .balign 128 + .global camera_model_start +camera_model_start: + .incbin CAMERA_MODEL_FILE + .global camera_model_end +camera_model_end: + .byte 0 + + .section .note.GNU-stack, "", @progbits diff --git a/examples/espressif/camera_stream/partitions.csv b/examples/espressif/camera_stream/partitions.csv new file mode 100644 index 00000000000..195f4193a6a --- /dev/null +++ b/examples/espressif/camera_stream/partitions.csv @@ -0,0 +1,3 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +factory, app, factory, 0x10000, 15M diff --git a/examples/espressif/camera_stream/sdkconfig.defaults b/examples/espressif/camera_stream/sdkconfig.defaults new file mode 100644 index 00000000000..da0678a871c --- /dev/null +++ b/examples/espressif/camera_stream/sdkconfig.defaults @@ -0,0 +1,18 @@ +CONFIG_IDF_TARGET="esp32s3" +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y +CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y +CONFIG_ESP_TASK_WDT_TIMEOUT_S=30 diff --git a/examples/espressif/camera_stream/viewer.py b/examples/espressif/camera_stream/viewer.py new file mode 100644 index 00000000000..4996151aed3 --- /dev/null +++ b/examples/espressif/camera_stream/viewer.py @@ -0,0 +1,789 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Live viewer for the ESP32-S3 camera inference stream (apps/camera_stream). + +The board prints text lines on its USB-Serial-JTAG port: + #CAM ... stream configuration + #CROP x= y= side= centre crop fed to the model, in camera pixels + #RES seq= pre_us= infer_us= cycle_us= top=idx:prob,... + #JPG kind=cam|input seq= res= w= h= enc_us= len= crc= b64=... + +Terminal dashboard (stdlib only; --preview draws the frame with Pillow): + python3 tools/camera_viewer.py --port /dev/ttyACM0 [--preview] + +Chrome dashboard on http://127.0.0.1:8765 (stdlib only): + python3 tools/camera_viewer.py --web + +Automated check that prints a JSON summary and exits: + python3 tools/camera_viewer.py --duration 30 --summary-json out.json +""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import fcntl +import http.server +import io +import json +import os +import re +import select +import shutil +import socketserver +import struct +import sys +import termios +import threading +import time +import tty +import zlib +from pathlib import Path + +TOOLS_DIR = Path(__file__).resolve().parent +TIOCM_DTR = 0x002 +TIOCM_RTS = 0x004 +INJECTED_LOG = re.compile(rb"cam_hal: [^\r\n]{0,96}\r\n") + + +def load_labels(path: Path) -> list[str]: + if path.is_file(): + return [line.strip() for line in path.read_text().splitlines()] + return [] + + +def parse_fields(text: str) -> dict[str, str]: + fields = {} + for item in text.split(): + if "=" in item: + key, value = item.split("=", 1) + fields[key] = value + return fields + + +class SerialPort: + def __init__(self, path: str): + self.fd = os.open(path, os.O_RDWR | os.O_NOCTTY) + tty.setraw(self.fd) + attributes = termios.tcgetattr(self.fd) + attributes[4] = attributes[5] = termios.B115200 + attributes[6][termios.VMIN] = 1 + attributes[6][termios.VTIME] = 0 + termios.tcsetattr(self.fd, termios.TCSANOW, attributes) + + def _set_lines(self, dtr: bool, rts: bool) -> None: + bits = struct.unpack( + "I", fcntl.ioctl(self.fd, termios.TIOCMGET, struct.pack("I", 0)) + )[0] + bits = (bits | TIOCM_DTR) if dtr else (bits & ~TIOCM_DTR) + bits = (bits | TIOCM_RTS) if rts else (bits & ~TIOCM_RTS) + fcntl.ioctl(self.fd, termios.TIOCMSET, struct.pack("I", bits)) + + def reset(self) -> None: + self._set_lines(dtr=False, rts=True) + time.sleep(0.1) + self._set_lines(dtr=False, rts=False) + + def read(self) -> bytes: + ready, _, _ = select.select([self.fd], [], [], 0.5) + if not ready: + return b"\0" + return os.read(self.fd, 65536) + + def close(self) -> None: + try: + os.close(self.fd) + except OSError: + pass + + +class StreamState: + def __init__(self, labels: list[str]): + self.labels = labels + self.lock = threading.Lock() + self.changed = threading.Condition(self.lock) + self.version = 0 + self.config: dict[str, str] = {} + self.crop: dict[str, int] = {} + self.result: dict | None = None + self.frames: dict[str, dict] = {} + self.logs = collections.deque(maxlen=12) + self.errors = collections.deque(maxlen=12) + self.result_times = collections.deque(maxlen=20) + self.frame_times = collections.deque(maxlen=60) + self.byte_times = collections.deque(maxlen=400) + self.infer_history = collections.deque(maxlen=120) + self.counts = collections.Counter() + self.started = time.time() + self.first_result_at: float | None = None + self.camera_log = "" + self.link = "starting" + + def label(self, index: int) -> str: + if 0 <= index < len(self.labels): + return self.labels[index] + return f"class {index}" + + def note_camera_log(self, messages: list[bytes]) -> None: + with self.lock: + for message in messages: + text = message.decode("utf-8", errors="replace").strip() + key = ( + "camera_overruns" + if "FB-OVF" in text or "FB-SIZE" in text + else "camera_logs" + ) + self.counts[key] += 1 + self.camera_log = text + + def add_bytes(self, count: int) -> None: + with self.lock: + self.byte_times.append((time.time(), count)) + + def handle_line(self, raw: bytes) -> None: + line = raw.decode("utf-8", errors="replace").rstrip("\r\n") + if not line: + return + now = time.time() + if line.startswith("#JPG "): + self._handle_jpeg(line, now) + return + with self.lock: + if line.startswith("#RES "): + fields = parse_fields(line[5:]) + top = [] + for item in fields.get("top", "").split(","): + if ":" in item: + index, probability = item.split(":", 1) + top.append( + { + "index": int(index), + "label": self.label(int(index)), + "probability": float(probability), + } + ) + self.result = { + "seq": int(fields.get("seq", 0)), + "pre_ms": int(fields.get("pre_us", 0)) / 1000.0, + "infer_ms": int(fields.get("infer_us", 0)) / 1000.0, + "cycle_ms": int(fields.get("cycle_us", 0)) / 1000.0, + "latency_ms": int(fields.get("latency_us", 0)) / 1000.0, + "decode_ms": int(fields.get("dec_us", 0)) / 1000.0, + "decode_errors": int(fields.get("dec_err", 0)), + "dropped": int(fields.get("dropped", 0)), + "usb_fail": int(fields.get("usb_fail", 0)), + "top": top, + "host_time": now, + } + self.result_times.append(now) + self.infer_history.append(self.result["infer_ms"]) + self.counts["results"] += 1 + if self.first_result_at is None: + self.first_result_at = now + elif line.startswith("#CAM "): + self.config = parse_fields(line[5:]) + elif line.startswith("#CROP "): + self.crop = {k: int(v) for k, v in parse_fields(line[6:]).items()} + elif line.startswith("#ERR"): + self.errors.append(line) + self.counts["device_errors"] += 1 + else: + self.logs.append(line) + self.version += 1 + self.changed.notify_all() + + def _handle_jpeg(self, line: str, now: float) -> None: + head, _, payload = line.partition(" b64=") + fields = parse_fields(head[5:]) + try: + data = base64.b64decode(payload.strip(), validate=True) + valid = len(data) == int(fields["len"]) and zlib.crc32(data) == int( + fields["crc"], 16 + ) + except (ValueError, KeyError): + valid = False + with self.lock: + if not valid: + self.counts["corrupt_frames"] += 1 + return + kind = fields.get("kind", "cam") + self.frames[kind] = { + "jpeg": data, + "seq": int(fields.get("seq", 0)), + "res": int(fields.get("res", 0)), + "width": int(fields.get("w", 0)), + "height": int(fields.get("h", 0)), + "encode_ms": int(fields.get("enc_us", 0)) / 1000.0, + "host_time": now, + } + self.counts[f"{kind}_frames"] += 1 + if kind == "cam": + self.frame_times.append(now) + self.version += 1 + self.changed.notify_all() + + @staticmethod + def _rate(times: collections.deque) -> float: + if len(times) < 2 or times[-1] == times[0]: + return 0.0 + return (len(times) - 1) / (times[-1] - times[0]) + + def stats(self) -> dict: + now = time.time() + window = [(t, n) for t, n in self.byte_times if now - t <= 3.0] + throughput = sum(n for _, n in window) / 3.0 if window else 0.0 + result = self.result + history = list(self.infer_history) + return { + "inference_fps": round(self._rate(self.result_times), 3), + "preview_fps": round(self._rate(self.frame_times), 2), + "link_kbps": round(throughput / 1024.0, 1), + "result_age_ms": ( + round((now - result["host_time"]) * 1000.0) if result else None + ), + "infer_ms_mean": round(sum(history) / len(history), 1) if history else None, + "infer_ms_min": min(history) if history else None, + "infer_ms_max": max(history) if history else None, + "counts": dict(self.counts), + "latency_ms": result["latency_ms"] if result else None, + "last_camera_log": self.camera_log, + "uptime_s": round(now - self.started, 1), + "link": self.link, + "config": self.config, + "crop": self.crop, + } + + def snapshot(self) -> dict: + with self.lock: + frame = self.frames.get("cam") + return { + "stats": self.stats(), + "result": self.result, + "frame": ( + {k: v for k, v in frame.items() if k != "jpeg"} if frame else None + ), + "infer_history": list(self.infer_history), + "errors": list(self.errors), + "logs": list(self.logs), + } + + +def set_link(state: StreamState, text: str) -> None: + with state.lock: + state.link = text + state.logs.append(f"viewer: {text}") + state.version += 1 + state.changed.notify_all() + + +def reader_thread( # noqa: C901 + path: str, reset: bool, pause_file: Path, state: StreamState, raw_log +) -> None: + pending = bytearray() + port = None + while True: + if port is None: + if pause_file.exists(): + if state.link != "paused for flashing": + set_link(state, "paused for flashing") + time.sleep(0.3) + continue + try: + port = SerialPort(path) + except OSError: + if state.link != "waiting for board": + set_link(state, "waiting for board") + time.sleep(0.5) + continue + if reset: + port.reset() + reset = False + pending.clear() + set_link(state, "connected") + if pause_file.exists(): + port.close() + port = None + continue + try: + chunk = port.read() + except OSError: + chunk = b"" + if not chunk: + port.close() + port = None + time.sleep(0.3) + continue + if chunk == b"\0": + continue + if raw_log is not None: + raw_log.write(chunk) + state.add_bytes(len(chunk)) + pending += chunk + if b"cam_hal: " in pending: + injected = INJECTED_LOG.findall(pending) + if injected: + pending = bytearray(INJECTED_LOG.sub(b"", pending)) + state.note_camera_log(injected) + while True: + newline = pending.find(b"\n") + if newline < 0: + break + line = bytes(pending[: newline + 1]) + del pending[: newline + 1] + state.handle_line(line) + if len(pending) > 4 * 1024 * 1024: + pending.clear() + + +PAGE = r""" + + +ESP32-S3 Live Inference + + +

ESP32-S3 live inference

+
connecting
+
+
+

Camera preview

+
camera stream
+
waiting for first result
+
+
+
-
inferences / s
+
-
inference ms
+
-
preprocess ms
+
-
preview fps
+
USB KB/s
+
-
capture to result ms
+
+
+
+
+

Top 5

+
+
+
+

What the model sees

+
model input
+
+
+

Inference latency

+ +
+
+

Device log

+

+    
+
+
+ + +""" + + +def make_handler(state: StreamState): # noqa: C901 + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args) -> None: + pass + + def _send(self, code: int, content_type: str, body: bytes) -> None: + self.send_response(code) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + path = self.path.split("?", 1)[0] + try: + if path == "/": + self._send(200, "text/html; charset=utf-8", PAGE.encode()) + elif path == "/state.json": + self._send( + 200, "application/json", json.dumps(self._snapshot()).encode() + ) + elif path in ("/frame.jpg", "/input.jpg"): + kind = "cam" if path == "/frame.jpg" else "input" + with state.lock: + frame = state.frames.get(kind) + if frame is None: + self._send(404, "text/plain", b"no frame yet") + else: + self._send(200, "image/jpeg", frame["jpeg"]) + elif path == "/stream.mjpg": + self._mjpeg() + elif path == "/events": + self._events() + else: + self._send(404, "text/plain", b"not found") + except (BrokenPipeError, ConnectionResetError): + pass + + def _snapshot(self) -> dict: + snapshot = state.snapshot() + with state.lock: + frame = state.frames.get("input") + snapshot["input_seq"] = frame["seq"] if frame else None + return snapshot + + def _mjpeg(self) -> None: + self.send_response(200) + self.send_header( + "Content-Type", "multipart/x-mixed-replace; boundary=frame" + ) + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + last = None + while True: + with state.changed: + state.changed.wait_for( + lambda last=last: state.frames.get("cam") is not None + and state.frames["cam"]["seq"] != last, + timeout=5.0, + ) + frame = state.frames.get("cam") + if frame is None or frame["seq"] == last: + continue + last = frame["seq"] + jpeg = frame["jpeg"] + self.wfile.write( + b"--frame\r\nContent-Type: image/jpeg\r\nContent-Length: " + + str(len(jpeg)).encode() + + b"\r\n\r\n" + + jpeg + + b"\r\n" + ) + self.wfile.flush() + + def _events(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + while True: + with state.changed: + state.changed.wait(timeout=0.5) + self.wfile.write( + b"data: " + json.dumps(self._snapshot()).encode() + b"\n\n" + ) + self.wfile.flush() + time.sleep(0.2) + + return Handler + + +class ThreadingServer(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def terminal_frame(jpeg: bytes, columns: int) -> list[str]: + try: + from PIL import Image + except ImportError: + return ["(install Pillow for --preview)"] + image = Image.open(io.BytesIO(jpeg)).convert("RGB") + rows = max(2, int(columns * image.height / image.width) // 2 * 2) + image = image.resize((columns, rows)) + pixels = image.load() + lines = [] + for y in range(0, rows, 2): + parts = [] + for x in range(columns): + top = pixels[x, y] + bottom = pixels[x, y + 1] + parts.append( + f"\x1b[38;2;{top[0]};{top[1]};{top[2]}m\x1b[48;2;{bottom[0]};{bottom[1]};{bottom[2]}m▀" + ) + lines.append("".join(parts) + "\x1b[0m") + return lines + + +def run_terminal(state: StreamState, args) -> None: + interactive = sys.stdout.isatty() and not args.plain + last_seq = None + deadline = time.time() + args.duration if args.duration else None + if interactive: + sys.stdout.write("\x1b[?25l\x1b[2J") + try: + while deadline is None or time.time() < deadline: + time.sleep(0.25 if interactive else 0.1) + snap = state.snapshot() + result = snap["result"] + stats = snap["stats"] + if not interactive: + if result and result["seq"] != last_seq: + last_seq = result["seq"] + top = ", ".join( + f"{t['label']} {t['probability'] * 100:.1f}%" + for t in result["top"][:3] + ) + print( + f"#{result['seq']:4d} infer {result['infer_ms']:6.1f} ms pre {result['pre_ms']:5.1f} ms " + f"latency {result['latency_ms']:6.1f} ms " + f"{stats['inference_fps']:.2f} inf/s preview {stats['preview_fps']:.1f} fps " + f"{stats['link_kbps']:.0f} KB/s | {top}", + flush=True, + ) + continue + width = shutil.get_terminal_size((100, 40)).columns + out = [ + "\x1b[H", + f"\x1b[1mESP32-S3 live inference\x1b[0m {state.config.get('model_input', '')} " + f"dual_core={state.config.get('dual_core', '?')}\x1b[K", + "\x1b[K", + ] + if args.preview: + with state.lock: + frame = state.frames.get("cam") + if frame: + out += list( + terminal_frame( + frame["jpeg"], min(args.preview_columns, width - 2) + ) + ) + out.append("\x1b[K") + if result: + out.append( + f"result #{result['seq']} inference {result['infer_ms']:.1f} ms preprocess " + f"{result['pre_ms']:.1f} ms cycle {result['cycle_ms']:.0f} ms " + f"capture-to-result {result['latency_ms']:.0f} ms\x1b[K" + ) + bar_width = max(10, min(40, width - 50)) + for t in result["top"]: + filled = int(round(t["probability"] * bar_width)) + out.append( + f" {t['label'][:30]:30s} {'#' * filled}{'.' * (bar_width - filled)} " + f"{t['probability'] * 100:5.1f}%\x1b[K" + ) + else: + out.append( + "waiting for the first result (the board runs its self-test first)\x1b[K" + ) + counts = stats["counts"] + out.append("\x1b[K") + out.append( + f"inference {stats['inference_fps']:.2f}/s preview {stats['preview_fps']:.1f} fps " + f"USB {stats['link_kbps']:.0f} KB/s result age {stats['result_age_ms']} ms\x1b[K" + ) + out.append( + f"frames {counts.get('cam_frames', 0)} corrupt {counts.get('corrupt_frames', 0)} " + f"camera overruns {counts.get('camera_overruns', 0)} " + f"device errors {counts.get('device_errors', 0)}\x1b[K" + ) + for line in snap["errors"][-3:] + snap["logs"][-4:]: + out.append(f"\x1b[2m{line[:width - 1]}\x1b[0m\x1b[K") + out.append("\x1b[J") + sys.stdout.write("\n".join(out)) + sys.stdout.flush() + except KeyboardInterrupt: + pass + finally: + if interactive: + sys.stdout.write("\x1b[0m\x1b[?25h\n") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--port", default=os.environ.get("ESP_PORT", "/dev/ttyACM0")) + parser.add_argument( + "--labels", type=Path, default=TOOLS_DIR / "imagenet_classes.txt" + ) + parser.add_argument( + "--reset", action="store_true", help="reset the board after opening the port" + ) + parser.add_argument("--web", action="store_true", help="serve the Chrome dashboard") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--http-port", type=int, default=8765) + parser.add_argument( + "--preview", action="store_true", help="draw the camera frame in the terminal" + ) + parser.add_argument("--preview-columns", type=int, default=64) + parser.add_argument( + "--plain", + action="store_true", + help="one line per result instead of a dashboard", + ) + parser.add_argument( + "--duration", type=float, default=0.0, help="exit after this many seconds" + ) + parser.add_argument( + "--summary-json", type=Path, help="write final statistics here on exit" + ) + parser.add_argument("--raw-log", type=Path, help="save the raw serial stream") + parser.add_argument( + "--pause-file", + type=Path, + default=TOOLS_DIR / "build" / ".viewer_pause", + help="the serial port is released while this file exists (flash.sh creates it)", + ) + args = parser.parse_args() + + state = StreamState(load_labels(args.labels)) + raw_log = open(args.raw_log, "wb") if args.raw_log else None + threading.Thread( + target=reader_thread, + args=(args.port, args.reset, args.pause_file, state, raw_log), + daemon=True, + ).start() + + if args.web: + server = ThreadingServer((args.host, args.http_port), make_handler(state)) + threading.Thread(target=server.serve_forever, daemon=True).start() + print(f"Chrome dashboard: http://{args.host}:{args.http_port}/", flush=True) + + run_terminal(state, args) + + with state.lock: + summary = state.stats() + summary["last_result"] = state.result + if args.summary_json: + args.summary_json.write_text(json.dumps(summary, indent=2)) + if args.duration: + print(json.dumps(summary, indent=2)) + if raw_log: + raw_log.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/espnn_bundle.py b/examples/espressif/espnn_bundle.py new file mode 100644 index 00000000000..1eee34d3537 --- /dev/null +++ b/examples/espressif/espnn_bundle.py @@ -0,0 +1,153 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +On-device test bundles for examples/espressif/espnn_runner. + +Bundle (little-endian), magic "XTB1" for XTS3 int8 graphs or "XTP1" for .pte programs: + magic u32 version=1 u32 count + count x { char name[32]; u32 blob_offset; u32 blob_size; u32 vectors_offset; u32 vectors_size } + payloads aligned to 128 bytes + +XTV2 vectors (XTS3 graphs; inputs and outputs in device layout, bit-exact): + "XTV2" u32 version=2 u32 samples u32 inputs u32 outputs + u32 input_bytes[inputs] u32 output_bytes[outputs] + samples x { int8 inputs..., int8 expected outputs... } + +XTE1 vectors (.pte programs; ExecuTorch tensors in logical layout): + "XTE1" u32 version=1 u32 samples u32 inputs u32 outputs f32 min_sqnr_db u32 flags + {u32 scalar_type, u32 numel} x (inputs + outputs) + samples x { inputs..., expected outputs... } + flags bit 0: outputs must be bit-identical; bit 1: outputs are stored as CRC32 only. +""" + +from __future__ import annotations + +import math +import struct +import zlib +from pathlib import Path + +import numpy as np +import torch + +from executorch.backends.espressif.reference import esp_nn_reference as reference + +NAME_BYTES = 32 +ALIGNMENT = 128 +FLAG_EXACT = 1 +FLAG_CRC = 2 +SCALAR_TYPES = {torch.float32: 6, torch.int8: 1, torch.int64: 4, torch.int32: 3} + + +def sqnr_db(reference_values: torch.Tensor, actual: torch.Tensor) -> float: + reference_values = reference_values.to(torch.float64) + noise = (reference_values - actual.to(torch.float64)).pow(2).sum().item() + if noise == 0: + return math.inf + return 10 * math.log10(reference_values.pow(2).sum().item() / noise) + + +def xtv2_vectors(graph, samples: list[list[np.ndarray]]) -> bytes: + """Quantize float samples and record the bit-exact outputs of the ESP-NN reference.""" + interpreter = reference.ReferenceInterpreter(graph) + input_tensors = [graph.tensors[i] for i in interpreter.inputs] + output_tensors = [graph.tensors[i] for i in interpreter.outputs] + input_bytes = [int(np.prod(reference.device_shape(t))) for t in input_tensors] + output_bytes = [int(np.prod(reference.device_shape(t))) for t in output_tensors] + body = bytearray() + for sample in samples: + device_inputs = [ + reference.to_device_layout(t, reference.quantize(t, np.asarray(value))) + for t, value in zip(input_tensors, sample) + ] + outputs = interpreter.run_outputs(device_inputs) + for value in device_inputs + outputs: + body += value.astype(np.int8).tobytes() + header = struct.pack( + "<4sIIII", b"XTV2", 2, len(samples), len(input_tensors), len(output_tensors) + ) + sizes = struct.pack( + f"<{len(input_bytes) + len(output_bytes)}I", *input_bytes, *output_bytes + ) + return header + sizes + bytes(body) + + +def xte1_vectors(rows, exact: bool, min_sqnr_db: float, crc: bool = False) -> bytes: + """rows: list of (inputs, outputs), each a list of tensors.""" + first_inputs, first_outputs = rows[0] + descriptors = [ + (SCALAR_TYPES[t.dtype], t.numel()) + for t in list(first_inputs) + list(first_outputs) + ] + flags = (FLAG_EXACT if exact else 0) | (FLAG_CRC if exact and crc else 0) + header = struct.pack( + "<4sIIIIfI", + b"XTE1", + 1, + len(rows), + len(first_inputs), + len(first_outputs), + float(min_sqnr_db), + flags, + ) + table = b"".join(struct.pack(" int: + """entries: list of (name, blob_bytes, vectors_bytes). Returns the bundle size in bytes.""" + if magic not in (b"XTB1", b"XTP1"): + raise ValueError(f"Unknown bundle magic {magic!r}") + payload = bytearray(b"\0" * (12 + len(entries) * (NAME_BYTES + 16))) + records = [] + for name, blob, vectors in entries: + payload.extend(b"\0" * ((-len(payload)) % ALIGNMENT)) + blob_offset = len(payload) + payload += blob + payload.extend(b"\0" * ((-len(payload)) % ALIGNMENT)) + vectors_offset = len(payload) + payload += vectors + records.append((name, blob_offset, len(blob), vectors_offset, len(vectors))) + header = struct.pack("<4sII", magic, 1, len(records)) + table = b"".join( + struct.pack(f"<{NAME_BYTES}sIIII", name.encode()[: NAME_BYTES - 1], *rest) + for name, *rest in records + ) + payload[: len(header) + len(table)] = header + table + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(bytes(payload)) + return len(payload) + + +def read_bundle(path: Path) -> tuple[bytes, list[tuple[str, bytes, bytes]]]: + data = path.read_bytes() + magic, version, count = struct.unpack_from("<4sII", data, 0) + entries = [] + for index in range(count): + name, blob_offset, blob_size, vectors_offset, vectors_size = struct.unpack_from( + f"<{NAME_BYTES}sIIII", data, 12 + index * (NAME_BYTES + 16) + ) + entries.append( + ( + name.rstrip(b"\0").decode(), + data[blob_offset : blob_offset + blob_size], + data[vectors_offset : vectors_offset + vectors_size], + ) + ) + return magic, entries + + +__all__ = ["read_bundle", "sqnr_db", "write_bundle", "xte1_vectors", "xtv2_vectors"] diff --git a/examples/espressif/espnn_runner/CMakeLists.txt b/examples/espressif/espnn_runner/CMakeLists.txt new file mode 100644 index 00000000000..90ffb639948 --- /dev/null +++ b/examples/espressif/espnn_runner/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +cmake_minimum_required(VERSION 3.16) + +get_filename_component( + EXECUTORCH_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE +) +set(EXTRA_COMPONENT_DIRS "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime") + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(espnn_runner) diff --git a/examples/espressif/espnn_runner/README.md b/examples/espressif/espnn_runner/README.md new file mode 100644 index 00000000000..5f88a53d905 --- /dev/null +++ b/examples/espressif/espnn_runner/README.md @@ -0,0 +1,49 @@ +# ESP-NN runner + +ESP-IDF application that runs a test bundle on an Espressif board, checks every +model against the host (bit-exact for int8 graphs, bit-exact or SQNR for +ExecuTorch programs with CPU operators) and times it. + +Bundles are created by `../aot_espnn_compiler.py` (one model) or +`../scripts/make_test_bundles.py` (the backend's operator test models): + +| Bundle | Contents | Needs | +|---|---|---| +| `.xtb` (XTB1) | XTS3 int8 graphs run by the standalone `espnn_runtime` interpreter | nothing else | +| `.xtp` (XTP1) | ExecuTorch `.pte` programs, ESP-NN delegate plus portable CPU kernels | ExecuTorch cross-compiled for the chip, `CONFIG_ESPNN_EXECUTORCH_DELEGATE=y` | + +## Build and run + +From an activated ESP-IDF v5.1+ shell: + +```bash +cd examples/espressif/espnn_runner +python ../scripts/make_test_bundles.py # bundles/op_suite.xtb, bundles/pte_suite.xtp + +# Standalone int8 graphs (any ESP32 chip) +idf.py set-target esp32s3 # or esp32 +idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults" menuconfig # ESP-NN runner -> bundle path +idf.py -p /dev/ttyACM0 flash monitor + +# ExecuTorch programs (needs ../scripts/build_executorch_esp.sh esp32s3 first) +idf.py -B build-et -DSDKCONFIG=build-et/sdkconfig \ + -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.executorch" \ + -DESPNN_EXECUTORCH_BUILD_DIR=$EXECUTORCH_ROOT/cmake-out-esp-esp32s3 build flash monitor +``` + +`../scripts/run_bundle.py BUNDLE --port PORT` does the same (target detection, +build, flash, capture) and prints a summary. + +Output, one line per model: + +``` +CHIP target=esp32s3 cores=2 revision=2 cpu_mhz=240 flash_mb=16 psram_kb=8192 ... +SUITE CASE 1/11 name=conv-act-pool-linear ops=9 ... exact=3 ... ansi=1 us=2444 min_us=... status=PASS +ET CASE 5/7 name=tiny-transformer ... delegate_us=2288 cpu_us=6040 sqnr_db=41.6 ... status=PASS +ESPNN RUNNER: PASS +``` + +`sdkconfig.defaults.esp32s3` assumes 16 MB flash and octal PSRAM (for example +ESP32-S3-DevKitC-1 N16R8). For other modules change the flash size, partition +table and PSRAM mode. Boards without PSRAM still boot (`CONFIG_SPIRAM_IGNORE_NOTFOUND`) +and run models whose activations fit in internal RAM. diff --git a/examples/espressif/espnn_runner/main/CMakeLists.txt b/examples/espressif/espnn_runner/main/CMakeLists.txt new file mode 100644 index 00000000000..56c8efdc9c5 --- /dev/null +++ b/examples/espressif/espnn_runner/main/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +set(runner_sources "main.cpp" "int8_suite.cpp" "bundle_data.S") +if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) + list(APPEND runner_sources "pte_suite.cpp") +endif() + +idf_component_register( + SRCS + ${runner_sources} + INCLUDE_DIRS + "." + REQUIRES + espnn_runtime + esp_timer + heap + esp_rom + spi_flash +) + +idf_build_get_property(project_dir PROJECT_DIR) +if(IS_ABSOLUTE "${CONFIG_ESPNN_RUNNER_BUNDLE}") + set(bundle "${CONFIG_ESPNN_RUNNER_BUNDLE}") +else() + set(bundle "${project_dir}/${CONFIG_ESPNN_RUNNER_BUNDLE}") +endif() +if(NOT EXISTS "${bundle}") + message( + FATAL_ERROR + "Test bundle ${bundle} not found. Create one with " + "examples/espressif/aot_espnn_compiler.py or examples/espressif/scripts/make_*.py " + "and set CONFIG_ESPNN_RUNNER_BUNDLE." + ) +endif() +set_source_files_properties( + "bundle_data.S" + PROPERTIES COMPILE_DEFINITIONS "ESPNN_BUNDLE_FILE=\"${bundle}\"" + OBJECT_DEPENDS "${bundle}" +) diff --git a/examples/espressif/espnn_runner/main/Kconfig.projbuild b/examples/espressif/espnn_runner/main/Kconfig.projbuild new file mode 100644 index 00000000000..733f6646986 --- /dev/null +++ b/examples/espressif/espnn_runner/main/Kconfig.projbuild @@ -0,0 +1,51 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +menu "ESP-NN runner" + + config ESPNN_RUNNER_BUNDLE + string "Test bundle to embed (XTB1 int8 graphs or XTP1 .pte programs)" + default "bundles/op_suite.xtb" + help + Absolute, or relative to this project directory. + + config ESPNN_RUNNER_BENCH_ITERATIONS + int "Timed iterations per model after the correctness check" + range 0 1000 + default 10 + + config ESPNN_RUNNER_VERIFY_ANSI + bool "Also run each int8 graph once with the ESP-NN ANSI reference kernels" + default y + + config ESPNN_RUNNER_DUAL_CORE + bool "Split ESP-NN kernels across both cores (int8 graphs)" + default n + depends on !FREERTOS_UNICORE + + config ESPNN_RUNNER_CONSTANTS_TO_PSRAM + bool "Copy int8 graph weights from flash into PSRAM when they fit" + default y + depends on SPIRAM + + config ESPNN_RUNNER_METHOD_POOL_KB + int "ExecuTorch method allocator pool (KB)" + default 1024 + depends on ESPNN_EXECUTORCH_DELEGATE + + config ESPNN_RUNNER_TEMP_POOL_KB + int "ExecuTorch temporary allocator pool (KB)" + default 512 + depends on ESPNN_EXECUTORCH_DELEGATE + + config ESPNN_RUNNER_STACK_KB + int "Runner task stack (KB)" + default 48 if ESPNN_EXECUTORCH_DELEGATE + default 16 + +endmenu diff --git a/examples/espressif/espnn_runner/main/bundle_data.S b/examples/espressif/espnn_runner/main/bundle_data.S new file mode 100644 index 00000000000..e16fce020f5 --- /dev/null +++ b/examples/espressif/espnn_runner/main/bundle_data.S @@ -0,0 +1,20 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + .section .rodata.espnn_bundle, "a" + .balign 128 + .global espnn_bundle_start +espnn_bundle_start: + .incbin ESPNN_BUNDLE_FILE + .global espnn_bundle_end +espnn_bundle_end: + .byte 0 + + .section .note.GNU-stack, "", @progbits diff --git a/examples/espressif/espnn_runner/main/int8_suite.cpp b/examples/espressif/espnn_runner/main/int8_suite.cpp new file mode 100644 index 00000000000..a40ab037b4e --- /dev/null +++ b/examples/espressif/espnn_runner/main/int8_suite.cpp @@ -0,0 +1,324 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +#include +#include +#include + +#include + +namespace executorch { +namespace examples { +namespace espressif { + +using ::executorch::backends::espressif::Int8Interpreter; +using ::executorch::backends::espressif::Int8RuntimeOptions; +using ::executorch::backends::espressif::Int8TensorInfo; +using ::executorch::backends::espressif::KernelMode; +using ::executorch::backends::espressif::MAX_GRAPH_IO; +using ::executorch::backends::espressif::read_le_u32; + +namespace { + +constexpr size_t BUNDLE_HEADER_SIZE = 12; +constexpr size_t BUNDLE_NAME_BYTES = 32; +constexpr size_t BUNDLE_RECORD_SIZE = BUNDLE_NAME_BYTES + 16; +constexpr size_t XTV2_HEADER_SIZE = 20; + +bool in_bounds(size_t offset, size_t length, size_t total) { + return offset <= total && length <= total - offset; +} + +struct VectorsV2 { + uint32_t samples; + uint32_t inputs; + uint32_t outputs; + uint32_t input_bytes[MAX_GRAPH_IO]; + uint32_t output_bytes[MAX_GRAPH_IO]; + size_t stride; + const uint8_t* data; +}; + +bool parse_vectors_v2(const uint8_t* data, size_t size, VectorsV2& v) { + if (size < XTV2_HEADER_SIZE || std::memcmp(data, "XTV2", 4) != 0 || + read_le_u32(data + 4) != 2) { + return false; + } + v.samples = read_le_u32(data + 8); + v.inputs = read_le_u32(data + 12); + v.outputs = read_le_u32(data + 16); + if (v.samples == 0 || v.inputs == 0 || v.outputs == 0 || + v.inputs > MAX_GRAPH_IO || v.outputs > MAX_GRAPH_IO) + return false; + const size_t table = XTV2_HEADER_SIZE + 4U * (v.inputs + v.outputs); + if (size < table) + return false; + v.stride = 0; + for (uint32_t i = 0; i < v.inputs; ++i) { + v.input_bytes[i] = read_le_u32(data + XTV2_HEADER_SIZE + 4 * i); + v.stride += v.input_bytes[i]; + } + for (uint32_t i = 0; i < v.outputs; ++i) { + v.output_bytes[i] = + read_le_u32(data + XTV2_HEADER_SIZE + 4 * (v.inputs + i)); + v.stride += v.output_bytes[i]; + } + v.data = data + table; + return v.stride != 0 && (size - table) / v.stride >= v.samples && + table + v.stride * v.samples == size; +} + +struct CaseResult { + uint32_t exact_samples; + uint32_t mismatched; + int32_t worst; + bool ansi_exact; + int64_t total_us; + int64_t bench_min_us; + int64_t bench_max_us; + int64_t bench_total_us; + uint32_t bench_iterations; +}; + +uint32_t compare(const int8_t* a, const int8_t* b, size_t n, int32_t& worst) { + uint32_t mismatches = 0; + for (size_t i = 0; i < n; ++i) { + int32_t d = static_cast(a[i]) - b[i]; + if (d < 0) + d = -d; + if (d != 0) + ++mismatches; + if (d > worst) + worst = d; + } + return mismatches; +} + +bool run_sample( + Int8Interpreter& interpreter, + const VectorsV2& v, + uint32_t index, + uint32_t& mismatched, + int32_t& worst) { + const uint8_t* sample = v.data + v.stride * index; + for (uint32_t i = 0; i < v.inputs; ++i) { + std::memcpy(interpreter.input_data(i), sample, v.input_bytes[i]); + sample += v.input_bytes[i]; + } + if (!interpreter.invoke()) + return false; + mismatched = 0; + for (uint32_t o = 0; o < v.outputs; ++o) { + mismatched += compare( + interpreter.output_data(o), + reinterpret_cast(sample), + v.output_bytes[o], + worst); + sample += v.output_bytes[o]; + } + return true; +} + +bool run_case( + const char* name, + const uint8_t* blob, + size_t blob_size, + const uint8_t* vectors, + size_t vectors_size, + uint32_t index, + uint32_t count) { + VectorsV2 v{}; + if (!parse_vectors_v2(vectors, vectors_size, v)) { + std::printf( + "SUITE CASE %u/%u name=%s status=FAIL reason=vectors\n", + static_cast(index), + static_cast(count), + name); + return false; + } + Int8Interpreter* interpreter = new (std::nothrow) Int8Interpreter(); + Int8RuntimeOptions options{}; +#if CONFIG_ESPNN_RUNNER_DUAL_CORE + options.dual_core = true; +#endif +#if CONFIG_ESPNN_RUNNER_CONSTANTS_TO_PSRAM + options.copy_constants_to_psram = true; +#endif + if (interpreter == nullptr || !interpreter->load(blob, blob_size, options)) { + std::printf( + "SUITE CASE %u/%u name=%s status=FAIL reason=load\n", + static_cast(index), + static_cast(count), + name); + delete interpreter; + return false; + } + bool shapes_ok = interpreter->input_count() == v.inputs && + interpreter->output_count() == v.outputs; + for (uint32_t i = 0; shapes_ok && i < v.inputs; ++i) { + Int8TensorInfo info{}; + shapes_ok = + interpreter->input_info(i, info) && info.bytes == v.input_bytes[i]; + } + for (uint32_t o = 0; shapes_ok && o < v.outputs; ++o) { + Int8TensorInfo info{}; + shapes_ok = + interpreter->output_info(o, info) && info.bytes == v.output_bytes[o]; + } + if (!shapes_ok) { + std::printf( + "SUITE CASE %u/%u name=%s status=FAIL reason=io-shapes\n", + static_cast(index), + static_cast(count), + name); + delete interpreter; + return false; + } + CaseResult result{}; + bool ok = true; + for (uint32_t s = 0; ok && s < v.samples; ++s) { + uint32_t mismatched = 0; + ok = run_sample(*interpreter, v, s, mismatched, result.worst); + result.mismatched += mismatched; + if (ok && mismatched == 0) + ++result.exact_samples; + result.total_us += interpreter->last_invoke_us(); + } + result.ansi_exact = true; +#if CONFIG_ESPNN_RUNNER_VERIFY_ANSI + if (ok) { + uint32_t mismatched = 0; + int32_t worst = 0; + interpreter->set_kernel_mode(KernelMode::REFERENCE); + ok = run_sample(*interpreter, v, 0, mismatched, worst); + interpreter->set_kernel_mode(KernelMode::OPTIMIZED); + result.ansi_exact = ok && mismatched == 0; + } +#endif + result.bench_min_us = INT64_MAX; + for (uint32_t i = 0; ok && i < CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS; ++i) { + ok = interpreter->invoke(); + const int64_t elapsed = interpreter->last_invoke_us(); + result.bench_total_us += elapsed; + if (elapsed < result.bench_min_us) + result.bench_min_us = elapsed; + if (elapsed > result.bench_max_us) + result.bench_max_us = elapsed; + ++result.bench_iterations; + } + if (result.bench_iterations == 0) + result.bench_min_us = 0; + const bool pass = + ok && result.exact_samples == v.samples && result.ansi_exact; + std::printf( + "SUITE CASE %u/%u name=%s ops=%u inputs=%u outputs=%u samples=%u exact=%u " + "mismatched=%u worst=%ld ansi=%d us=%lld min_us=%lld max_us=%lld iterations=%u " + "arena=%u arena_internal=%d constants_in_ram=%d blob=%u status=%s\n", + static_cast(index), + static_cast(count), + name, + static_cast(interpreter->graph().header.num_operations), + static_cast(v.inputs), + static_cast(v.outputs), + static_cast(v.samples), + static_cast(result.exact_samples), + static_cast(result.mismatched), + static_cast(result.worst), + result.ansi_exact ? 1 : 0, + static_cast( + result.bench_iterations + ? result.bench_total_us / result.bench_iterations + : result.total_us / v.samples), + static_cast(result.bench_min_us), + static_cast(result.bench_max_us), + static_cast(result.bench_iterations), + static_cast(interpreter->arena_bytes()), + interpreter->arena_in_internal_ram() ? 1 : 0, + interpreter->constants_copied() ? 1 : 0, + static_cast(blob_size), + pass ? "PASS" : "FAIL"); + if (!pass) + interpreter->print_profile(); + delete interpreter; + return pass; +} + +} // namespace + +bool run_int8_suite(const uint8_t* data, size_t size) { + std::printf("\n========================================\n"); + std::printf(" ESP-NN INT8 OPERATOR SUITE\n"); + std::printf("========================================\n"); + if (!is_int8_bundle(data, size) || read_le_u32(data + 4) != 1) { + std::printf("SUITE RESULT: FAIL reason=bundle\n"); + return false; + } + const uint32_t count = read_le_u32(data + 8); + if (count == 0 || + !in_bounds( + BUNDLE_HEADER_SIZE, + static_cast(count) * BUNDLE_RECORD_SIZE, + size)) { + std::printf("SUITE RESULT: FAIL reason=bundle-table\n"); + return false; + } + uint32_t passed = 0; + const int64_t started = esp_timer_get_time(); + for (uint32_t i = 0; i < count; ++i) { + const uint8_t* record = + data + BUNDLE_HEADER_SIZE + static_cast(i) * BUNDLE_RECORD_SIZE; + char name[BUNDLE_NAME_BYTES + 1]{}; + std::memcpy(name, record, BUNDLE_NAME_BYTES); + const uint32_t blob_offset = read_le_u32(record + BUNDLE_NAME_BYTES); + const uint32_t blob_size = read_le_u32(record + BUNDLE_NAME_BYTES + 4); + const uint32_t vectors_offset = read_le_u32(record + BUNDLE_NAME_BYTES + 8); + const uint32_t vectors_size = read_le_u32(record + BUNDLE_NAME_BYTES + 12); + if (!in_bounds(blob_offset, blob_size, size) || + !in_bounds(vectors_offset, vectors_size, size)) { + std::printf( + "SUITE CASE %u/%u name=%s status=FAIL reason=bounds\n", + static_cast(i + 1), + static_cast(count), + name); + continue; + } + if (run_case( + name, + data + blob_offset, + blob_size, + data + vectors_offset, + vectors_size, + i + 1, + count)) + ++passed; + } + std::printf( + "Internal RAM free %u, PSRAM free %u, suite time %lld ms\n", + static_cast( + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM)), + static_cast((esp_timer_get_time() - started) / 1000)); + std::printf( + "SUITE RESULT: %s cases=%u passed=%u\n", + passed == count ? "PASS" : "FAIL", + static_cast(count), + static_cast(passed)); + return passed == count; +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/espnn_runner/main/main.cpp b/examples/espressif/espnn_runner/main/main.cpp new file mode 100644 index 00000000000..12bdd031f74 --- /dev/null +++ b/examples/espressif/espnn_runner/main/main.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +extern const uint8_t espnn_bundle_start[]; +extern const uint8_t espnn_bundle_end[]; +} + +namespace { + +void print_chip() { + esp_chip_info_t chip{}; + esp_chip_info(&chip); + uint32_t flash_bytes = 0; + esp_flash_get_size(nullptr, &flash_bytes); + std::printf( + "CHIP target=%s cores=%d revision=%d cpu_mhz=%d flash_mb=%u psram_kb=%u " + "internal_free_kb=%u idf=%s\n", + CONFIG_IDF_TARGET, + chip.cores, + chip.revision, + CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ, + static_cast(flash_bytes / (1024 * 1024)), + static_cast(heap_caps_get_total_size(MALLOC_CAP_SPIRAM) / 1024), + static_cast( + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) / + 1024), + esp_get_idf_version()); +} + +void runner_task(void*) { + print_chip(); + const size_t size = reinterpret_cast(espnn_bundle_end) - + reinterpret_cast(espnn_bundle_start); + bool pass = false; + if (executorch::examples::espressif::is_int8_bundle( + espnn_bundle_start, size)) { + pass = executorch::examples::espressif::run_int8_suite( + espnn_bundle_start, size); + } else if (executorch::examples::espressif::is_pte_bundle( + espnn_bundle_start, size)) { +#if CONFIG_ESPNN_EXECUTORCH_DELEGATE + pass = executorch::examples::espressif::run_pte_suite( + espnn_bundle_start, size); +#else + std::printf( + "ERROR: this bundle holds .pte programs; enable CONFIG_ESPNN_EXECUTORCH_DELEGATE\n"); +#endif + } else { + std::printf( + "ERROR: unrecognised bundle; expected XTB1 (int8 graphs) or XTP1 (.pte programs)\n"); + } + std::printf("ESPNN RUNNER: %s\n", pass ? "PASS" : "FAIL"); + while (true) + vTaskDelay(pdMS_TO_TICKS(10000)); +} + +} // namespace + +extern "C" void app_main() { + xTaskCreatePinnedToCore( + runner_task, + "espnn_runner", + CONFIG_ESPNN_RUNNER_STACK_KB * 1024, + nullptr, + 5, + nullptr, + 0); +} diff --git a/examples/espressif/espnn_runner/main/pte_suite.cpp b/examples/espressif/espnn_runner/main/pte_suite.cpp new file mode 100644 index 00000000000..58cb7cb32af --- /dev/null +++ b/examples/espressif/espnn_runner/main/pte_suite.cpp @@ -0,0 +1,494 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +using executorch::aten::ScalarType; +using executorch::aten::Tensor; +using executorch::extension::BufferDataLoader; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; + +namespace executorch { +namespace examples { +namespace espressif { + +using ::executorch::backends::espressif::DelegateStats; +using ::executorch::backends::espressif::delegate_stats; +using ::executorch::backends::espressif::read_le_u32; +using ::executorch::backends::espressif::register_espnn_backend; +using ::executorch::backends::espressif::reset_delegate_stats; + +namespace { + +constexpr size_t BUNDLE_HEADER_SIZE = 12; +constexpr size_t BUNDLE_NAME_BYTES = 32; +constexpr size_t BUNDLE_RECORD_SIZE = BUNDLE_NAME_BYTES + 16; +constexpr size_t XTE1_HEADER_SIZE = 28; +constexpr uint32_t XTE1_FLAG_EXACT = 1; +constexpr uint32_t XTE1_FLAG_CRC = 2; +constexpr size_t MAX_IO = 16; +constexpr size_t MAX_PLANNED_BUFFERS = 8; + +bool in_bounds(size_t offset, size_t length, size_t total) { + return offset <= total && length <= total - offset; +} + +size_t element_size(uint32_t dtype) { + if (dtype == static_cast(ScalarType::Float)) + return 4; + if (dtype == static_cast(ScalarType::Char)) + return 1; + if (dtype == static_cast(ScalarType::Long)) + return 8; + if (dtype == static_cast(ScalarType::Int)) + return 4; + return 0; +} + +struct Vectors { + uint32_t samples; + uint32_t inputs; + uint32_t outputs; + float min_sqnr_db; + uint32_t flags; + uint32_t dtype[2 * MAX_IO]; + uint32_t numel[2 * MAX_IO]; + size_t stride; + const uint8_t* data; +}; + +bool parse_vectors(const uint8_t* data, size_t size, Vectors& v) { + if (size < XTE1_HEADER_SIZE || std::memcmp(data, "XTE1", 4) != 0 || + read_le_u32(data + 4) != 1) { + return false; + } + v.samples = read_le_u32(data + 8); + v.inputs = read_le_u32(data + 12); + v.outputs = read_le_u32(data + 16); + const uint32_t sqnr_bits = read_le_u32(data + 20); + std::memcpy(&v.min_sqnr_db, &sqnr_bits, sizeof(float)); + v.flags = read_le_u32(data + 24); + if (v.samples == 0 || v.inputs == 0 || v.outputs == 0 || v.inputs > MAX_IO || + v.outputs > MAX_IO) { + return false; + } + const size_t table = XTE1_HEADER_SIZE + 8U * (v.inputs + v.outputs); + if (size < table) + return false; + v.stride = 0; + for (uint32_t i = 0; i < v.inputs + v.outputs; ++i) { + v.dtype[i] = read_le_u32(data + XTE1_HEADER_SIZE + 8 * i); + v.numel[i] = read_le_u32(data + XTE1_HEADER_SIZE + 8 * i + 4); + const size_t bytes = element_size(v.dtype[i]) * v.numel[i]; + if (bytes == 0) + return false; + v.stride += (i >= v.inputs && (v.flags & XTE1_FLAG_CRC) != 0) ? 4U : bytes; + } + v.data = data + table; + return (size - table) / v.stride >= v.samples && + table + v.stride * v.samples == size; +} + +struct Pools { + uint8_t* method_pool = nullptr; + uint8_t* temp_pool = nullptr; + uint8_t* planned[MAX_PLANNED_BUFFERS]{}; + size_t planned_count = 0; + size_t planned_bytes = 0; + + ~Pools() { + if (method_pool != nullptr) + heap_caps_free(method_pool); + if (temp_pool != nullptr) + heap_caps_free(temp_pool); + for (size_t i = 0; i < planned_count; ++i) { + if (planned[i] != nullptr) + heap_caps_free(planned[i]); + } + } +}; + +void* psram(size_t bytes) { + void* pointer = + heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (pointer == nullptr) + pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_8BIT); + return pointer; +} + +struct OutputScore { + double signal; + double noise; + float max_abs; + uint32_t mismatched; +}; + +void score_output( + const Tensor& actual, + uint32_t dtype, + const uint8_t* expected, + size_t count, + OutputScore& score) { + if (dtype == static_cast(ScalarType::Float)) { + const float* got = actual.const_data_ptr(); + for (size_t i = 0; i < count; ++i) { + float want = 0.0f; + std::memcpy(&want, expected + 4 * i, sizeof(float)); + const double difference = static_cast(got[i]) - want; + score.signal += static_cast(want) * want; + score.noise += difference * difference; + const float magnitude = static_cast(std::fabs(difference)); + if (magnitude > score.max_abs) + score.max_abs = magnitude; + if (got[i] != want) + ++score.mismatched; + } + return; + } + const uint8_t* got = static_cast(actual.const_data_ptr()); + const size_t bytes = count * element_size(dtype); + for (size_t i = 0; i < bytes; ++i) { + if (got[i] != expected[i]) + ++score.mismatched; + } +} + +bool run_case( + const char* name, + const uint8_t* pte, + size_t pte_size, + const uint8_t* vectors, + size_t vectors_size, + uint32_t index, + uint32_t count) { + Vectors v{}; + if (!parse_vectors(vectors, vectors_size, v)) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=vectors\n", + static_cast(index), + static_cast(count), + name); + return false; + } + reset_delegate_stats(); + const int64_t load_started = esp_timer_get_time(); + BufferDataLoader loader(pte, pte_size); + Result program = Program::load(&loader); + if (!program.ok()) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=program-load error=0x%x\n", + static_cast(index), + static_cast(count), + name, + static_cast(program.error())); + return false; + } + Result meta = program->method_meta("forward"); + if (!meta.ok() || meta->num_memory_planned_buffers() > MAX_PLANNED_BUFFERS) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=method-meta\n", + static_cast(index), + static_cast(count), + name); + return false; + } + Pools pools; + const size_t method_bytes = + static_cast(CONFIG_ESPNN_RUNNER_METHOD_POOL_KB) * 1024U; + const size_t temp_bytes = + static_cast(CONFIG_ESPNN_RUNNER_TEMP_POOL_KB) * 1024U; + pools.method_pool = static_cast(psram(method_bytes)); + pools.temp_pool = static_cast(psram(temp_bytes)); + Span spans[MAX_PLANNED_BUFFERS]; + pools.planned_count = meta->num_memory_planned_buffers(); + bool allocated = pools.method_pool != nullptr && pools.temp_pool != nullptr; + for (size_t i = 0; allocated && i < pools.planned_count; ++i) { + const size_t bytes = + static_cast(meta->memory_planned_buffer_size(i).get()); + pools.planned[i] = static_cast(psram(bytes == 0 ? 16 : bytes)); + allocated = pools.planned[i] != nullptr; + spans[i] = Span(pools.planned[i], bytes); + pools.planned_bytes += bytes; + } + if (!allocated) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=memory\n", + static_cast(index), + static_cast(count), + name); + return false; + } + MemoryAllocator method_allocator( + static_cast(method_bytes), pools.method_pool); + MemoryAllocator temp_allocator( + static_cast(temp_bytes), pools.temp_pool); + HierarchicalAllocator planned( + Span>(spans, pools.planned_count)); + MemoryManager memory(&method_allocator, &planned, &temp_allocator); + bool pass = false; + { + Result method = program->load_method("forward", &memory); + if (!method.ok()) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=method-load error=0x%x\n", + static_cast(index), + static_cast(count), + name, + static_cast(method.error())); + return false; + } + const int64_t load_us = esp_timer_get_time() - load_started; + if (method->inputs_size() != v.inputs || + method->outputs_size() != v.outputs) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=io-count\n", + static_cast(index), + static_cast(count), + name); + return false; + } + EValue inputs[MAX_IO]; + OutputScore scores[MAX_IO]{}; + int64_t total_us = 0; + int64_t fastest = INT64_MAX; + bool ok = true; + for (uint32_t s = 0; ok && s < v.samples; ++s) { + const uint8_t* sample = v.data + v.stride * s; + ok = method->get_inputs(inputs, v.inputs) == Error::Ok; + for (uint32_t i = 0; ok && i < v.inputs; ++i) { + Tensor tensor = inputs[i].toTensor(); + const size_t bytes = element_size(v.dtype[i]) * v.numel[i]; + ok = static_cast(tensor.scalar_type()) == v.dtype[i] && + tensor.nbytes() == bytes; + if (ok) + std::memcpy(tensor.mutable_data_ptr(), sample, bytes); + sample += bytes; + } + if (!ok) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=input-shape\n", + static_cast(index), + static_cast(count), + name); + return false; + } + const int64_t started = esp_timer_get_time(); + const Error status = method->execute(); + const int64_t elapsed = esp_timer_get_time() - started; + if (status != Error::Ok) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=execute error=0x%x\n", + static_cast(index), + static_cast(count), + name, + static_cast(status)); + return false; + } + total_us += elapsed; + if (elapsed < fastest) + fastest = elapsed; + for (uint32_t o = 0; o < v.outputs; ++o) { + const uint32_t slot = v.inputs + o; + const Tensor tensor = method->get_output(o).toTensor(); + const size_t bytes = element_size(v.dtype[slot]) * v.numel[slot]; + if (static_cast(tensor.scalar_type()) != v.dtype[slot] || + tensor.nbytes() != bytes) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=output-shape\n", + static_cast(index), + static_cast(count), + name); + return false; + } + if ((v.flags & XTE1_FLAG_CRC) != 0) { + const uint32_t crc = esp_rom_crc32_le( + 0, static_cast(tensor.const_data_ptr()), bytes); + if (crc != read_le_u32(sample)) + ++scores[o].mismatched; + sample += 4; + continue; + } + score_output(tensor, v.dtype[slot], sample, v.numel[slot], scores[o]); + sample += bytes; + } + } + int64_t bench_min = INT64_MAX; + int64_t bench_max = 0; + int64_t bench_total = 0; + uint32_t bench_iterations = 0; + for (uint32_t i = 0; ok && i < CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS; ++i) { + const int64_t started = esp_timer_get_time(); + ok = method->execute() == Error::Ok; + const int64_t elapsed = esp_timer_get_time() - started; + bench_total += elapsed; + if (elapsed < bench_min) + bench_min = elapsed; + if (elapsed > bench_max) + bench_max = elapsed; + ++bench_iterations; + } + if (bench_iterations == 0) + bench_min = fastest; + double worst_sqnr = INFINITY; + float max_abs = 0.0f; + uint32_t mismatched = 0; + for (uint32_t o = 0; o < v.outputs; ++o) { + mismatched += scores[o].mismatched; + if (scores[o].max_abs > max_abs) + max_abs = scores[o].max_abs; + if (scores[o].noise > 0.0 && scores[o].signal > 0.0) { + const double sqnr = + 10.0 * std::log10(scores[o].signal / scores[o].noise); + if (sqnr < worst_sqnr) + worst_sqnr = sqnr; + } else if (scores[o].noise > 0.0) { + worst_sqnr = -INFINITY; + } + } + const DelegateStats stats = delegate_stats(); + const uint32_t runs = v.samples + bench_iterations; + const int64_t mean_us = bench_iterations ? bench_total / bench_iterations + : total_us / v.samples; + const int64_t delegate_us = stats.calls ? stats.total_us / runs : 0; + pass = (v.flags & XTE1_FLAG_EXACT) != 0 ? mismatched == 0 + : worst_sqnr >= v.min_sqnr_db; + std::printf( + "ET CASE %u/%u name=%s inputs=%u outputs=%u samples=%u delegates=%u " + "delegate_calls=%u us=%lld min_us=%lld max_us=%lld iterations=%u delegate_us=%lld " + "cpu_us=%lld sqnr_db=%.1f " + "max_abs=%.6g mismatched=%u exact_required=%d min_sqnr=%.1f load_us=%lld " + "planned=%u arena=%u status=%s\n", + static_cast(index), + static_cast(count), + name, + static_cast(v.inputs), + static_cast(v.outputs), + static_cast(v.samples), + static_cast(stats.delegates), + static_cast(stats.calls / runs), + static_cast(mean_us), + static_cast(bench_min < fastest ? bench_min : fastest), + static_cast(bench_iterations ? bench_max : mean_us), + static_cast(bench_iterations), + static_cast(delegate_us), + static_cast(mean_us - delegate_us), + std::isinf(worst_sqnr) ? (worst_sqnr > 0 ? 999.0 : -999.0) : worst_sqnr, + static_cast(max_abs), + static_cast(mismatched), + (v.flags & XTE1_FLAG_EXACT) != 0 ? 1 : 0, + static_cast(v.min_sqnr_db), + static_cast(load_us), + static_cast(pools.planned_bytes), + static_cast(stats.arena_bytes), + pass ? "PASS" : "FAIL"); + } + return pass; +} + +} // namespace + +bool run_pte_suite(const uint8_t* data, size_t size) { + std::printf("\n========================================\n"); + std::printf( + " EXECUTORCH RUNTIME SUITE (ESP-NN delegate + portable CPU kernels)\n"); + std::printf("========================================\n"); + executorch::runtime::runtime_init(); + if (!register_espnn_backend()) { + std::printf("ET RESULT: FAIL reason=backend-registration\n"); + return false; + } + if (!is_pte_bundle(data, size) || read_le_u32(data + 4) != 1) { + std::printf("ET RESULT: FAIL reason=bundle\n"); + return false; + } + const uint32_t count = read_le_u32(data + 8); + if (count == 0 || + !in_bounds( + BUNDLE_HEADER_SIZE, + static_cast(count) * BUNDLE_RECORD_SIZE, + size)) { + std::printf("ET RESULT: FAIL reason=bundle-table\n"); + return false; + } + uint32_t passed = 0; + const int64_t started = esp_timer_get_time(); + for (uint32_t i = 0; i < count; ++i) { + const uint8_t* record = + data + BUNDLE_HEADER_SIZE + static_cast(i) * BUNDLE_RECORD_SIZE; + char name[BUNDLE_NAME_BYTES + 1]{}; + std::memcpy(name, record, BUNDLE_NAME_BYTES); + const uint32_t pte_offset = read_le_u32(record + BUNDLE_NAME_BYTES); + const uint32_t pte_size = read_le_u32(record + BUNDLE_NAME_BYTES + 4); + const uint32_t vectors_offset = read_le_u32(record + BUNDLE_NAME_BYTES + 8); + const uint32_t vectors_size = read_le_u32(record + BUNDLE_NAME_BYTES + 12); + if (!in_bounds(pte_offset, pte_size, size) || + !in_bounds(vectors_offset, vectors_size, size)) { + std::printf( + "ET CASE %u/%u name=%s status=FAIL reason=bounds\n", + static_cast(i + 1), + static_cast(count), + name); + continue; + } + if (run_case( + name, + data + pte_offset, + pte_size, + data + vectors_offset, + vectors_size, + i + 1, + count)) { + ++passed; + } + std::printf( + "ET MEMORY after=%s internal_free=%u psram_free=%u\n", + name, + static_cast( + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM))); + } + std::printf( + "ET RESULT: %s cases=%u passed=%u time_ms=%lld\n", + passed == count ? "PASS" : "FAIL", + static_cast(count), + static_cast(passed), + static_cast((esp_timer_get_time() - started) / 1000)); + return passed == count; +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/espnn_runner/main/runner.h b/examples/espressif/espnn_runner/main/runner.h new file mode 100644 index 00000000000..a881dcf3218 --- /dev/null +++ b/examples/espressif/espnn_runner/main/runner.h @@ -0,0 +1,48 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include +#include + +namespace executorch { +namespace examples { +namespace espressif { + +/// Whether @p data is an XTB1 bundle of XTS3 int8 graphs. +inline bool is_int8_bundle(const uint8_t* data, size_t size) { + return data != nullptr && size >= 12 && std::memcmp(data, "XTB1", 4) == 0; +} + +/// Whether @p data is an XTP1 bundle of ExecuTorch programs. +inline bool is_pte_bundle(const uint8_t* data, size_t size) { + return data != nullptr && size >= 12 && std::memcmp(data, "XTP1", 4) == 0; +} + +/** + * Runs every graph in an XTB1 bundle, checks it against the bundled vectors + * and prints one SUITE CASE line per graph. + * + * @retval true Every case passed. + */ +bool run_int8_suite(const uint8_t* data, size_t size); +/** + * Loads every program in an XTP1 bundle with the ExecuTorch runtime, checks + * it against the bundled vectors and prints one ET CASE line per program. + * + * @retval true Every case passed. + */ +bool run_pte_suite(const uint8_t* data, size_t size); + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/espnn_runner/partitions_16mb.csv b/examples/espressif/espnn_runner/partitions_16mb.csv new file mode 100644 index 00000000000..195f4193a6a --- /dev/null +++ b/examples/espressif/espnn_runner/partitions_16mb.csv @@ -0,0 +1,3 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +factory, app, factory, 0x10000, 15M diff --git a/examples/espressif/espnn_runner/partitions_4mb.csv b/examples/espressif/espnn_runner/partitions_4mb.csv new file mode 100644 index 00000000000..e1a3fef0195 --- /dev/null +++ b/examples/espressif/espnn_runner/partitions_4mb.csv @@ -0,0 +1,3 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +factory, app, factory, 0x10000, 0x3F0000 diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults b/examples/espressif/espnn_runner/sdkconfig.defaults new file mode 100644 index 00000000000..84c7e8732ab --- /dev/null +++ b/examples/espressif/espnn_runner/sdkconfig.defaults @@ -0,0 +1,11 @@ +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4mb.csv" +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_IGNORE_NOTFOUND=y +CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 +CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768 +CONFIG_ESP_TASK_WDT_TIMEOUT_S=30 diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 new file mode 100644 index 00000000000..e2940388858 --- /dev/null +++ b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 @@ -0,0 +1,5 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_ESPTOOLPY_FLASHFREQ_40M=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4mb.csv" +CONFIG_SPIRAM_TYPE_AUTO=y +CONFIG_SPIRAM_SPEED_40M=y diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 new file mode 100644 index 00000000000..4d724480d1a --- /dev/null +++ b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 @@ -0,0 +1,9 @@ +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_16mb.csv" +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y +CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y diff --git a/examples/espressif/espnn_runner/sdkconfig.executorch b/examples/espressif/espnn_runner/sdkconfig.executorch new file mode 100644 index 00000000000..2e0962c8be4 --- /dev/null +++ b/examples/espressif/espnn_runner/sdkconfig.executorch @@ -0,0 +1 @@ +CONFIG_ESPNN_EXECUTORCH_DELEGATE=y diff --git a/examples/espressif/quantize_and_lower.py b/examples/espressif/quantize_and_lower.py new file mode 100644 index 00000000000..96224b71f50 --- /dev/null +++ b/examples/espressif/quantize_and_lower.py @@ -0,0 +1,54 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Minimal example: quantize a torchvision model to int8 with the ESP-NN +quantizer and lower it to an ExecuTorch program for Espressif chips. + + python examples/espressif/quantize_and_lower.py --output mobilenet_v2_espnn.pte +""" + +from __future__ import annotations + +import argparse + +import torch +import torchvision +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="mobilenet_v2_espnn.pte") + args = parser.parse_args() + + model = torchvision.models.mobilenet_v2(weights="DEFAULT").eval() + example_inputs = (torch.randn(1, 3, 224, 224),) + + exported = torch.export.export(model, example_inputs).module() + prepared = prepare_pt2e(exported, EspNNQuantizer()) + with torch.no_grad(): + for _ in range(16): + prepared(torch.randn(1, 3, 224, 224)) + quantized = convert_pt2e(prepared) + + program = torch.export.export(quantized, example_inputs) + edge, report = to_espnn_edge(program) + print( + f"{report['partitions']} EspNN partition(s), {report['delegated_ops']} delegated operators, " + f"{len(report['cpu_ops'])} operators left for the CPU" + ) + + with open(args.output, "wb") as file: + file.write(edge.to_executorch().buffer) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/examples/espressif/scripts/bench_models.py b/examples/espressif/scripts/bench_models.py new file mode 100644 index 00000000000..b63b60f219b --- /dev/null +++ b/examples/espressif/scripts/bench_models.py @@ -0,0 +1,195 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Model factories for examples/espressif/scripts/benchmark.py. + +Use with aot_espnn_compiler.py as --model examples/espressif/scripts/bench_models.py:. +Small models use fixed-seed weights: they measure latency and memory, not accuracy. +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def _randomize_batch_norm(model: nn.Module, seed: int) -> nn.Module: + generator = torch.Generator().manual_seed(seed) + for module in model.modules(): + if isinstance(module, nn.BatchNorm2d): + channels = module.num_features + module.running_mean.copy_(torch.randn(channels, generator=generator) * 0.1) + module.running_var.copy_( + torch.rand(channels, generator=generator) * 0.5 + 0.75 + ) + module.weight.data.copy_( + torch.rand(channels, generator=generator) * 0.5 + 0.75 + ) + module.bias.data.copy_(torch.randn(channels, generator=generator) * 0.1) + return model + + +class DSCNN(nn.Module): + """Keyword spotting DS-CNN (MLPerf Tiny style) on a 1x49x10 MFCC input.""" + + def __init__(self, num_classes: int = 12, width: int = 64): + super().__init__() + layers = [ + nn.Conv2d(1, width, (10, 4), stride=(2, 2), padding=(5, 1), bias=False), + nn.BatchNorm2d(width), + nn.ReLU(), + ] + for _ in range(4): + layers += [ + nn.Conv2d(width, width, 3, padding=1, groups=width, bias=False), + nn.BatchNorm2d(width), + nn.ReLU(), + nn.Conv2d(width, width, 1, bias=False), + nn.BatchNorm2d(width), + nn.ReLU(), + ] + self.features = nn.Sequential(*layers) + self.pool = nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = nn.Linear(width, num_classes) + + def forward(self, x): + return self.classifier(torch.flatten(self.pool(self.features(x)), 1)) + + +class BasicBlock(nn.Module): + def __init__(self, channels: int): + super().__init__() + self.conv1 = nn.Conv2d(channels, channels, 3, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(channels) + self.conv2 = nn.Conv2d(channels, channels, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(channels) + + def forward(self, x): + y = F.relu(self.bn1(self.conv1(x))) + return F.relu(self.bn2(self.conv2(y)) + x) + + +class TinyResNet(nn.Module): + """64x64 image classifier with residuals, grouped conv and concat.""" + + def __init__(self, num_classes: int = 10): + super().__init__() + self.stem = nn.Sequential( + nn.Conv2d(3, 24, 3, stride=2, padding=1, bias=False), + nn.BatchNorm2d(24), + nn.ReLU(), + ) + self.pool = nn.MaxPool2d(3, stride=2, padding=1) + self.block1 = BasicBlock(24) + self.down = nn.Sequential( + nn.Conv2d(24, 48, 3, stride=2, padding=1, groups=4, bias=False), + nn.BatchNorm2d(48), + nn.ReLU6(), + ) + self.branch_a = nn.Sequential(nn.Conv2d(48, 32, 1), nn.ReLU()) + self.branch_b = nn.Sequential(nn.Conv2d(48, 32, 3, padding=1), nn.ReLU()) + self.block2 = BasicBlock(64) + self.avg = nn.AvgPool2d(2) + self.head = nn.Sequential( + nn.Flatten(), + nn.Linear(64 * 4 * 4, 128), + nn.ReLU(), + nn.Linear(128, num_classes), + ) + + def forward(self, x): + x = self.block1(self.pool(self.stem(x))) + x = self.down(x) + x = self.block2(torch.cat([self.branch_a(x), self.branch_b(x)], dim=1)) + return self.head(self.avg(x)) + + +class TinyTransformer(nn.Module): + """Patch-embedding transformer: shows the cost of CPU fallback (LayerNorm, attention).""" + + def __init__(self, dim: int = 32, heads: int = 2, classes: int = 10): + super().__init__() + self.heads = heads + self.stem = nn.Conv2d(3, dim, 8, stride=8) + self.norm1 = nn.LayerNorm(dim) + self.qkv = nn.Linear(dim, 3 * dim) + self.proj = nn.Linear(dim, dim) + self.norm2 = nn.LayerNorm(dim) + self.mlp1 = nn.Linear(dim, 2 * dim) + self.mlp2 = nn.Linear(2 * dim, dim) + self.head = nn.Linear(dim, classes) + + def forward(self, x): + tokens = self.stem(x).flatten(2).transpose(1, 2) + n, t, d = tokens.shape + q, k, v = self.qkv(self.norm1(tokens)).chunk(3, dim=-1) + q, k, v = ( + z.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + for z in (q, k, v) + ) + attention = torch.softmax( + q @ k.transpose(-2, -1) / (d // self.heads) ** 0.5, dim=-1 + ) + tokens = tokens + self.proj((attention @ v).transpose(1, 2).reshape(n, t, d)) + tokens = tokens + self.mlp2(F.gelu(self.mlp1(self.norm2(tokens)))) + return self.head(tokens.mean(dim=1)) + + +class SegmentationOut(nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, x): + return self.model(x)["out"] + + +class DetectorHeads(nn.Module): + """SSDLite backbone and heads; anchor decoding and NMS stay on the host or app side.""" + + def __init__(self, model): + super().__init__() + self.backbone = model.backbone + self.head = model.head + + def forward(self, x): + head = self.head(list(self.backbone(x).values())) + return head["bbox_regression"], head["cls_logits"] + + +def ds_cnn_kws() -> nn.Module: + torch.manual_seed(1) + return _randomize_batch_norm(DSCNN(), 1).eval() + + +def tiny_resnet() -> nn.Module: + torch.manual_seed(0) + return _randomize_batch_norm(TinyResNet(), 0).eval() + + +def tiny_transformer() -> nn.Module: + torch.manual_seed(2) + return TinyTransformer().eval() + + +def lraspp_mobilenet_v3() -> nn.Module: + import torchvision + + return SegmentationOut( + torchvision.models.segmentation.lraspp_mobilenet_v3_large(weights="DEFAULT") + ).eval() + + +def ssdlite320_mobilenet_v3() -> nn.Module: + import torchvision + + return DetectorHeads( + torchvision.models.detection.ssdlite320_mobilenet_v3_large(weights="DEFAULT") + ).eval() diff --git a/examples/espressif/scripts/benchmark.py b/examples/espressif/scripts/benchmark.py new file mode 100644 index 00000000000..fc857aa1f61 --- /dev/null +++ b/examples/espressif/scripts/benchmark.py @@ -0,0 +1,520 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Benchmark the ESP-NN backend on a connected Espressif board. + +For each model in the zoo this script quantizes and lowers it with +aot_espnn_compiler.py, packs the results into bundles that fit the detected +chip's flash, builds and flashes examples/espressif/espnn_runner, and records +correctness (bit-exact or SQNR against the host) and latency (min/mean/max over +timed iterations). Results go to /.json and .md. + +Run inside an activated ESP-IDF shell with a Python that has executorch: + python examples/espressif/scripts/benchmark.py --port /dev/ttyACM0 \ + --images ~/data/imagenette2-160 --dashboard + +Models that need more RAM than the board has are reported as "did not fit". +ExecuTorch-mode models need ExecuTorch cross-compiled for the target +(build_executorch_esp.sh) and enough flash for the runtime (about 6 MB with +all portable kernels); pass --skip-executorch on 4 MB boards. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).resolve().parent +EXAMPLES_DIR = SCRIPTS_DIR.parent +sys.path.insert(0, str(SCRIPTS_DIR)) +sys.path.insert(0, str(EXAMPLES_DIR)) + +import espnn_board as board # noqa: E402 +from espnn_bundle import read_bundle, write_bundle # noqa: E402 + +ZOO = f"{SCRIPTS_DIR / 'bench_models.py'}" +IMAGENET = (["--mean", "0.485", "0.456", "0.406"], ["--std", "0.229", "0.224", "0.225"]) +FIRMWARE_ESTIMATE = {"standalone": 600 * 1024, "executorch": 7 * 1024 * 1024} + + +@dataclass +class BenchModel: + name: str + model: str + shapes: list[str] + mode: str + images: bool + description: str + extra: tuple[str, ...] = () + + +MODELS = [ + BenchModel( + "ds_cnn_kws", + f"{ZOO}:ds_cnn_kws", + ["1,1,49,10"], + "standalone", + False, + "Keyword spotting DS-CNN, 49x10 MFCC", + ), + BenchModel( + "tiny_resnet_64", + f"{ZOO}:tiny_resnet", + ["1,3,64,64"], + "standalone", + True, + "Small residual CNN, 64x64", + ), + BenchModel( + "mobilenet_v3_small_96", + "mobilenet_v3_small", + ["1,3,96,96"], + "standalone", + True, + "MobileNetV3-Small, ImageNet, 96x96", + ("--resize", "110"), + ), + BenchModel( + "mobilenet_v3_small_224", + "mobilenet_v3_small", + ["1,3,224,224"], + "standalone", + True, + "MobileNetV3-Small, ImageNet, 224x224", + ), + BenchModel( + "mobilenet_v2_224", + "mobilenet_v2", + ["1,3,224,224"], + "standalone", + True, + "MobileNetV2, ImageNet, 224x224", + ), + BenchModel( + "mobilenet_v3_large_224", + "mobilenet_v3_large", + ["1,3,224,224"], + "standalone", + True, + "MobileNetV3-Large, ImageNet, 224x224", + ), + BenchModel( + "efficientnet_b0_224", + "efficientnet_b0", + ["1,3,224,224"], + "standalone", + True, + "EfficientNet-B0, ImageNet, 224x224", + ), + BenchModel( + "lraspp_mobilenet_v3_seg", + f"{ZOO}:lraspp_mobilenet_v3", + ["1,3,224,224"], + "executorch", + True, + "LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224", + ("--crc-outputs",), + ), + BenchModel( + "ssdlite320_mobilenet_v3_det", + f"{ZOO}:ssdlite320_mobilenet_v3", + ["1,3,320,320"], + "executorch", + True, + "SSDLite320 MobileNetV3 detection heads, COCO, 320x320", + ("--crc-outputs", "--mean", "0.5", "0.5", "0.5", "--std", "0.5", "0.5", "0.5"), + ), + BenchModel( + "tiny_transformer_fallback", + f"{ZOO}:tiny_transformer", + ["1,3,64,64"], + "executorch", + False, + "Small vision transformer: LayerNorm and attention on the CPU fallback", + ), +] + + +def compile_model(spec: BenchModel, out_dir: Path, args) -> Path: + suffix = ".xtp" if spec.mode == "executorch" else ".xtb" + bundle = out_dir / f"{spec.name}{suffix}" + if ( + bundle.exists() + and bundle.with_suffix(".json").exists() + and not args.rebuild_models + ): + return bundle + command = [ + sys.executable, + str(EXAMPLES_DIR / "aot_espnn_compiler.py"), + "--model", + spec.model, + "--mode", + spec.mode, + "--output", + str(bundle), + "--name", + spec.name, + "--calib-samples", + str(args.calib_samples), + "--eval-samples", + str(args.eval_samples), + "--vector-samples", + "2", + ] + for shape in spec.shapes: + command += ["--input-shape", shape] + if spec.images and args.images: + command += [ + "--calib-dir", + str(Path(args.images) / "train"), + "--eval-dir", + str(Path(args.images) / "val"), + ] + if not any(flag in spec.extra for flag in ("--mean",)) and spec.images: + command += IMAGENET[0] + IMAGENET[1] + command += list(spec.extra) + print(f"[compile] {spec.name}", flush=True) + log = out_dir / f"{spec.name}.compile.log" + result = subprocess.run(command, capture_output=True, text=True) + log.write_text(result.stdout + result.stderr) + if result.returncode != 0: + raise RuntimeError(f"compiling {spec.name} failed; see {log}") + return bundle + + +def group_bundles(entries, budget: int): + groups, current, size = [], [], 0 + for entry in sorted(entries, key=lambda e: len(e[1]) + len(e[2])): + entry_size = len(entry[1]) + len(entry[2]) + 256 + if entry_size > budget: + yield None, entry + continue + if current and size + entry_size > budget: + groups.append(current) + current, size = [], 0 + current.append(entry) + size += entry_size + if current: + groups.append(current) + for group in groups: + yield group, None + + +def markdown(target: str, chip: dict, rows: list[dict], calib_samples: int) -> str: + lines = [ + f"# ESP-NN backend benchmarks: {target}", + "", + f"Board: {chip.get('target', target)}, {chip.get('cores', '?')} cores at " + f"{chip.get('cpu_mhz', '?')} MHz, {chip.get('flash_mb', '?')} MB flash, " + f"{chip.get('psram_kb', '?')} KB PSRAM, ESP-IDF {chip.get('idf', '?')}.", + "", + "| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy |", + "|---|---|---|---|---|---|---|---|---|---|", + ] + for row in rows: + lines.append( + "| " + + " | ".join( + str(row.get(k, "")) + for k in ( + "description", + "input", + "path", + "operators", + "size", + "arena", + "latency", + "fps", + "check", + "accuracy", + ) + ) + + " |" + ) + lines.append("") + lines.append( + "Latency is the model invocation only (no input preprocessing), timed over " + "repeated runs after a bit-exact (or SQNR) check against the host reference." + ) + lines.append("") + lines.append( + f"Host accuracy is the int8 model on the host (the board's output is bit-identical) " + f"after post-training quantization with {calib_samples} calibration images and no " + "QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with " + "random weights report top-1 agreement or SQNR against the float model instead." + ) + return "\n".join(lines) + "\n" + + +def summarize( + spec: BenchModel, + report: dict, + case: dict | None, + failure: str | None, + board_chip: dict, +) -> dict: + row = { + "name": spec.name, + "description": spec.description, + "input": " / ".join(spec.shapes), + } + if spec.mode == "standalone": + row["path"] = "standalone XTS3" + row["operators"] = f"{sum(report.get('operations', {}).values())} int8" + row["size"] = f"{report.get('blob_bytes', 0) / 1e6:.2f} MB" + else: + row["path"] = "ExecuTorch .pte" + row["operators"] = ( + f"{report.get('delegated_ops', 0)} delegated, " + f"{len(report.get('cpu_ops', []))} CPU" + ) + row["size"] = f"{report.get('pte_bytes', 0) / 1e6:.2f} MB" + evaluation = report.get("evaluation", {}) + if evaluation.get("labelled") and not spec.model.startswith(ZOO): + row["accuracy"] = ( + f"top-1 {evaluation['device_correct']}/{evaluation['labelled']} " + f"(float {evaluation['float_correct']}), agrees with float " + f"{evaluation['top1_agree_with_float']}/{evaluation['samples']}" + ) + elif evaluation.get("samples"): + row["accuracy"] = ( + f"top-1 agrees with float {evaluation['top1_agree_with_float']}/{evaluation['samples']}" + ) + else: + row["accuracy"] = f"{report.get('sqnr_vs_float_db', 0):.1f} dB vs float" + if failure or case is None: + row.update(latency="-", fps="-", arena="-", check=failure or "not run") + return row + mean_us = float(case.get("us", 0)) + row["latency"] = ( + f"{mean_us / 1000:.1f} ms ({float(case.get('min_us', mean_us)) / 1000:.1f})" + ) + row["fps"] = f"{1e6 / mean_us:.2f}" if mean_us else "-" + row["arena"] = f"{int(case.get('arena', 0)) / 1024:.0f} KB" + arena = re.search(r"allocate (\d+)-byte activation arena", case.get("error", "")) + if case.get("status") != "PASS" and arena: + free = board_chip.get("internal_free_kb", "?") if board_chip else "?" + row.update( + latency="-", + fps="-", + arena=f"{int(arena.group(1)) / 1024:.0f} KB", + check=( + f"does not fit RAM ({free} KB free, no PSRAM)" + if board_chip.get("psram_kb") == "0" + else "does not fit RAM" + ), + ) + return row + if case.get("status") != "PASS": + row["check"] = f"FAIL ({case.get('reason', 'mismatch')})" + elif case["kind"] == "standalone": + row["check"] = f"bit-exact {case.get('exact')}/{case.get('samples')}" + ( + ", ANSI too" if case.get("ansi") == "1" else "" + ) + elif case.get("exact_required") == "1": + row["check"] = "bit-exact" + else: + row["check"] = f"{float(case.get('sqnr_db', 0)):.1f} dB vs host" + if case["kind"] == "executorch" and case.get("delegate_us"): + row[ + "latency" + ] += f"; delegate {int(case['delegate_us']) / 1000:.1f}, CPU {max(int(case['cpu_us']), 0) / 1000:.1f}" + return row + + +def main() -> int: # noqa: C901 + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--port", default="/dev/ttyACM0") + parser.add_argument( + "--target", help="override the detected chip (esp32s3, esp32, ...)" + ) + parser.add_argument("--models", help="comma separated subset of model names") + parser.add_argument( + "--images", help="Imagenette-style folder with train/ and val/ for calibration" + ) + parser.add_argument("--calib-samples", type=int, default=64) + parser.add_argument("--eval-samples", type=int, default=32) + parser.add_argument("--iterations", type=int, default=10) + parser.add_argument("--output", type=Path, default=EXAMPLES_DIR / "benchmarks") + parser.add_argument( + "--work-dir", type=Path, default=EXAMPLES_DIR / "benchmarks" / "work" + ) + parser.add_argument("--rebuild-models", action="store_true") + parser.add_argument("--skip-executorch", action="store_true") + parser.add_argument("--executorch-build-dir", type=Path) + parser.add_argument("--executorch-source-dir", type=Path) + parser.add_argument( + "--dashboard", + action="store_true", + help="live results at http://127.0.0.1:8770/", + ) + parser.add_argument("--timeout", type=float, default=900) + args = parser.parse_args() + + chip = board.detect_chip(args.port) + target = args.target or chip.target + print( + f"[board] {chip.description} -> target {target}, {chip.flash_mb} MB flash", + flush=True, + ) + selected = [ + m for m in MODELS if not args.models or m.name in args.models.split(",") + ] + if args.skip_executorch: + selected = [m for m in selected if m.mode != "executorch"] + + dashboard = None + if args.dashboard: + import board_dashboard + + dashboard = board_dashboard.start(open_browser=True) + + model_dir = args.work_dir / "models" + model_dir.mkdir(parents=True, exist_ok=True) + reports = {} + bundles = {"standalone": [], "executorch": []} + for spec in selected: + try: + bundle = compile_model(spec, model_dir, args) + except RuntimeError as error: + print(f"[compile] {error}", flush=True) + reports[spec.name] = {"error": str(error)} + continue + reports[spec.name] = json.loads(bundle.with_suffix(".json").read_text())[0] + _, entries = read_bundle(bundle) + bundles[spec.mode].extend(entries) + + cases: dict[str, dict] = {} + failures: dict[str, str] = {} + board_chip: dict = {} + for mode, entries in bundles.items(): + if not entries: + continue + budget = chip.app_partition_bytes - FIRMWARE_ESTIMATE[mode] + build = board.RunnerBuild( + target=target, + executorch=mode == "executorch", + build_dir=args.work_dir / f"build-{target}-{mode}", + executorch_build_dir=args.executorch_build_dir, + executorch_source_dir=args.executorch_source_dir, + ) + index = -1 + for group, too_big in group_bundles(entries, budget): + if too_big is not None: + failures[too_big[0]] = f"does not fit {chip.flash_mb} MB flash" + continue + index += 1 + magic = b"XTP1" if mode == "executorch" else b"XTB1" + path = ( + args.work_dir + / f"{target}-{mode}-{index}{'.xtp' if mode == 'executorch' else '.xtb'}" + ) + write_bundle(path, magic, group) + names = [name for name, _, _ in group] + print(f"[board] {mode} group {index}: {', '.join(names)}", flush=True) + job = f"{mode}-{index}" + if dashboard: + dashboard.begin_job( + job, + f"{target} {mode} group {index}", + {name: {"host": reports.get(name, {})} for name in names}, + ) + try: + result = board.run_bundle( + build, + path, + args.port, + args.iterations, + args.timeout, + args.work_dir / "logs" / job, + on_line=(lambda line: dashboard.log(line)) if dashboard else None, + on_case=( + (lambda case, job=job: dashboard.case(job, case)) + if dashboard + else None + ), + on_status=( + (lambda status, job=job: dashboard.status(job, status)) + if dashboard + else None + ), + ) + except RuntimeError as error: + for name in names: + failures[name] = str(error).splitlines()[0] + if dashboard: + dashboard.status(job, "build failed") + continue + board_chip = result.chip or board_chip + for case in result.cases: + cases[case.get("name", "?")] = case + for name in names: + if name not in cases: + failures[name] = ( + "did not run (out of memory or crash; see board.log)" + ) + if dashboard: + dashboard.status(job, "PASS" if result.passed else "FAIL") + + rows = [] + for spec in selected: + report = reports.get(spec.name, {}) + if "error" in report: + rows.append( + { + "name": spec.name, + "description": spec.description, + "input": " / ".join(spec.shapes), + "check": "compile failed", + } + ) + continue + rows.append( + summarize( + spec, report, cases.get(spec.name), failures.get(spec.name), board_chip + ) + ) + args.output.mkdir(parents=True, exist_ok=True) + payload = { + "target": target, + "chip": board_chip or {"target": target}, + "esptool": chip.description, + "date": time.strftime("%Y-%m-%d"), + "iterations": args.iterations, + "rows": rows, + "raw_cases": cases, + "failures": failures, + } + (args.output / f"{target}.json").write_text(json.dumps(payload, indent=2)) + (args.output / f"{target}.md").write_text( + markdown(target, payload["chip"], rows, args.calib_samples) + ) + print((args.output / f"{target}.md").read_text()) + if dashboard: + dashboard.finish() + print("dashboard still serving at http://127.0.0.1:8770/ (Ctrl+C to stop)") + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + pass + return 0 if all(not failures.get(spec.name) for spec in selected) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/scripts/board_dashboard.py b/examples/espressif/scripts/board_dashboard.py new file mode 100644 index 00000000000..a24ad74dee1 --- /dev/null +++ b/examples/espressif/scripts/board_dashboard.py @@ -0,0 +1,308 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Live Chrome dashboard for on-board runs of examples/espressif/espnn_runner. + +Used by benchmark.py --dashboard; can also show a saved results file: + python examples/espressif/scripts/board_dashboard.py --results benchmarks/esp32s3.json +""" + +from __future__ import annotations + +import argparse +import http.server +import json +import socketserver +import threading +import time +import webbrowser +from pathlib import Path + + +class State: + def __init__(self): + self.lock = threading.Lock() + self.changed = threading.Condition(self.lock) + self.version = 0 + self.jobs: dict[str, dict] = {} + self.order: list[str] = [] + self.logs: list[str] = [] + self.running = False + + def touch(self): + self.version += 1 + self.changed.notify_all() + + def snapshot(self) -> dict: + with self.lock: + return { + "jobs": self.jobs, + "logs": self.logs[-200:], + "running": self.running, + "version": self.version, + "order": list(self.order), + } + + +class Dashboard: + """Thread-safe facade that benchmark.py calls while a run progresses.""" + + def __init__(self, state: State, url: str): + self.state = state + self.url = url + with state.lock: + state.running = True + state.touch() + + def begin_job(self, job: str, title: str, cases: dict) -> None: + with self.state.lock: + self.state.jobs[job] = { + "title": title, + "status": "queued", + "cases": cases, + "result_line": "", + } + if job not in self.state.order: + self.state.order.append(job) + self.state.touch() + + def status(self, job: str, status: str) -> None: + with self.state.lock: + self.state.jobs.setdefault(job, {"cases": {}})["status"] = status + self.state.touch() + + def case(self, job: str, fields: dict) -> None: + with self.state.lock: + cases = self.state.jobs.setdefault(job, {"cases": {}}).setdefault( + "cases", {} + ) + cases.setdefault(fields.get("name", "?"), {})["device"] = fields + self.state.touch() + + def log(self, line: str) -> None: + if not line.startswith( + ("SUITE", "ET ", "CHIP", "ESPNN", "ERROR", "Guru", "abort") + ): + return + with self.state.lock: + self.state.logs.append(line[:240]) + del self.state.logs[:-400] + self.state.touch() + + def finish(self) -> None: + with self.state.lock: + self.state.running = False + self.state.touch() + + +PAGE = r""" + + +ESP-NN Board Dashboard + + +

ESP-NN board dashboard

idle
+
+

Live log

+ +""" + + +def make_handler(state: State): + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_GET(self): + path = self.path.split("?", 1)[0] + try: + if path == "/": + body = PAGE.encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + elif path == "/state.json": + body = json.dumps(state.snapshot()).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + elif path == "/events": + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + seen = -1 + while True: + with state.changed: + state.changed.wait_for( + lambda seen=seen: state.version != seen, timeout=10 + ) + snapshot = state.snapshot() + seen = snapshot["version"] + self.wfile.write( + b"data: " + json.dumps(snapshot).encode() + b"\n\n" + ) + self.wfile.flush() + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + except (BrokenPipeError, ConnectionResetError): + pass + + return Handler + + +class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def start( + port: int = 8770, open_browser: bool = False, state: State | None = None +) -> Dashboard: + state = state or State() + server = Server(("127.0.0.1", port), make_handler(state)) + threading.Thread(target=server.serve_forever, daemon=True).start() + url = f"http://127.0.0.1:{port}/" + print(f"dashboard: {url}", flush=True) + if open_browser: + webbrowser.open(url) + return Dashboard(state, url) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--results", type=Path, required=True, help="benchmark.py JSON results" + ) + parser.add_argument("--port", type=int, default=8770) + args = parser.parse_args() + results = json.loads(args.results.read_text()) + dashboard = start(args.port, open_browser=True) + title = f"{results.get('target', '')} benchmark {results.get('date', '')}" + dashboard.begin_job("results", title, {}) + for case in results.get("raw_cases", {}).values(): + dashboard.case("results", case) + dashboard.status("results", "loaded") + dashboard.finish() + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/scripts/build_executorch_esp.sh b/examples/espressif/scripts/build_executorch_esp.sh new file mode 100755 index 00000000000..6d8c79e5074 --- /dev/null +++ b/examples/espressif/scripts/build_executorch_esp.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Cross-compile the ExecuTorch runtime, portable kernels and quantized kernels +# for an Espressif chip with the ESP-IDF toolchain. +# +# Run inside an activated ESP-IDF shell (. $IDF_PATH/export.sh): +# examples/espressif/scripts/build_executorch_esp.sh [esp32s3|esp32|esp32s2|esp32p4|...] +# +# Environment: +# EXECUTORCH_ROOT ExecuTorch source tree (default: this repository) +# ET_BUILD_DIR output directory (default: $EXECUTORCH_ROOT/cmake-out-esp-) +# CMAKE CMake >= 3.24 (ExecuTorch requirement) +# PYTHON_EXECUTABLE Python with executorch and torch installed (kernel registration codegen) +# ET_SELECT_OPS_LIST comma separated ops for a selective build, e.g. "aten::add.out,..." +# ET_SELECT_OPS_MODEL a .pte whose operators define the selective build +set -euo pipefail + +TARGET=${1:-${IDF_TARGET:-esp32s3}} +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +EXECUTORCH_ROOT=${EXECUTORCH_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} +BUILD_DIR=${ET_BUILD_DIR:-$EXECUTORCH_ROOT/cmake-out-esp-$TARGET} +CMAKE=${CMAKE:-cmake} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} + +if [ -z "${IDF_PATH:-}" ]; then + echo "build_executorch_esp: IDF_PATH is not set; run . \$IDF_PATH/export.sh first" >&2 + exit 1 +fi + +case "$TARGET" in + esp32|esp32s2|esp32s3) + TARGET_FLAGS="-mlongcalls -fno-builtin-memcpy -fno-builtin-memset -fno-builtin-bzero -mdisable-hardware-atomics" ;; + *) + TARGET_FLAGS="-fno-builtin-memcpy -fno-builtin-memset -fno-builtin-bzero" ;; +esac + +SELECT_ARGS=() +if [ -n "${ET_SELECT_OPS_LIST:-}" ]; then + SELECT_ARGS+=(-DEXECUTORCH_SELECT_OPS_LIST="$ET_SELECT_OPS_LIST") +fi +if [ -n "${ET_SELECT_OPS_MODEL:-}" ]; then + SELECT_ARGS+=(-DEXECUTORCH_SELECT_OPS_MODEL="$ET_SELECT_OPS_MODEL") +fi +if [ ${#SELECT_ARGS[@]} -eq 0 ]; then + SELECT_ARGS+=(-DEXECUTORCH_BUILD_PORTABLE_OPS=ON) +fi + +mkdir -p "$BUILD_DIR" +cd "$EXECUTORCH_ROOT" +"$CMAKE" --preset esp-baremetal -B "$BUILD_DIR" -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE="$IDF_PATH/tools/cmake/toolchain-$TARGET.cmake" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ + -DCMAKE_INSTALL_PREFIX="$BUILD_DIR" \ + -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=OFF \ + -DEXECUTORCH_ENABLE_LOGGING=ON \ + -DEXECUTORCH_BUILD_DEVTOOLS=OFF \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF \ + -DEXECUTORCH_BUILD_CPUINFO=OFF \ + -DEXECUTORCH_BUILD_XNNPACK=OFF \ + -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=OFF \ + -DEXECUTORCH_PAL_DEFAULT=minimal \ + "${SELECT_ARGS[@]}" + +# Outside an ESP-IDF project the toolchain file leaves its flag response +# files empty; ESP32/S2/S3 code needs -mlongcalls to link. +for flags_file in cflags cxxflags asmflags; do + printf '%s\n' $TARGET_FLAGS > "$BUILD_DIR/toolchain/$flags_file" +done +"$CMAKE" --build "$BUILD_DIR" -j"$(nproc)" --target install +echo "build_executorch_esp: libraries in $BUILD_DIR/lib, headers in $BUILD_DIR/include" diff --git a/examples/espressif/scripts/espnn_board.py b/examples/espressif/scripts/espnn_board.py new file mode 100644 index 00000000000..7982f655a0f --- /dev/null +++ b/examples/espressif/scripts/espnn_board.py @@ -0,0 +1,297 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Board helpers for the ESP-NN examples: chip detection, ESP-IDF build and +flash of examples/espressif/espnn_runner, serial capture with reset, and +parsing of the runner's result lines. Needs an activated ESP-IDF shell. +""" + +from __future__ import annotations + +import fcntl +import os +import re +import select +import struct +import subprocess +import sys +import termios +import time +import tty +from dataclasses import dataclass, field +from pathlib import Path + +EXAMPLES_DIR = Path(__file__).resolve().parents[1] +RUNNER_DIR = EXAMPLES_DIR / "espnn_runner" +EXECUTORCH_ROOT = EXAMPLES_DIR.parents[1] +TIOCM_DTR = 0x002 +TIOCM_RTS = 0x004 +FIELDS = re.compile(r"(\w+)=(\S+)") +APP_PARTITION_BYTES = {4: 0x3F0000, 8: 0x7F0000, 16: 15 * 1024 * 1024} + + +def idf_command() -> list[str]: + idf_path = os.environ.get("IDF_PATH") + if not idf_path: + raise RuntimeError("IDF_PATH is not set; run . $IDF_PATH/export.sh first") + python = os.environ.get("IDF_PYTHON_ENV_PATH") + interpreter = f"{python}/bin/python" if python else "python3" + return [interpreter, f"{idf_path}/tools/idf.py"] + + +def tool_env() -> dict[str, str]: + """ESP-IDF tools run in their own venv; a PYTHONPATH pointing at the torch + environment would shadow its pinned packages (click, esptool, ...).""" + return { + k: v for k, v in os.environ.items() if k not in ("PYTHONPATH", "PYTHONHOME") + } + + +def esptool_command() -> list[str]: + python = os.environ.get("IDF_PYTHON_ENV_PATH") + return [f"{python}/bin/python" if python else "python3", "-m", "esptool"] + + +@dataclass +class ChipInfo: + target: str + description: str + flash_mb: int + + @property + def app_partition_bytes(self) -> int: + return APP_PARTITION_BYTES.get(self.flash_mb, APP_PARTITION_BYTES[4]) + + +def detect_chip(port: str) -> ChipInfo: + output = subprocess.run( + esptool_command() + ["--port", port, "flash_id"], + capture_output=True, + text=True, + timeout=60, + env=tool_env(), + ).stdout + match = re.search(r"Chip (?:is|type:)\s*(ESP32[-\w]*)([^\n]*)", output) + if not match: + raise RuntimeError(f"Could not identify the chip on {port}:\n{output[-800:]}") + name = match.group(1).upper() + family = re.match(r"ESP32-(S2|S3|C2|C3|C5|C6|C61|H2|H4|P4)\b", name) + target = f"esp32{family.group(1).lower()}" if family else "esp32" + flash = re.search(r"Detected flash size:\s*(\d+)MB", output) + return ChipInfo( + target=target, + description=(match.group(1) + match.group(2)).strip(), + flash_mb=int(flash.group(1)) if flash else 4, + ) + + +def set_sdkconfig(path: Path, values: dict[str, str]) -> None: + lines = path.read_text().splitlines() if path.exists() else [] + remaining = dict(values) + output = [] + for line in lines: + key = line.split("=", 1)[0].replace("# ", "").split(" is not set")[0].strip() + if key in remaining: + output.append(f"{key}={remaining.pop(key)}") + else: + output.append(line) + output.extend(f"{key}={value}" for key, value in remaining.items()) + path.write_text("\n".join(output) + "\n") + + +@dataclass +class RunnerBuild: + target: str + executorch: bool + build_dir: Path + executorch_build_dir: Path | None = None + executorch_source_dir: Path | None = None + extra_defaults: list[Path] = field(default_factory=list) + + def cmake_args(self) -> list[str]: + defaults = [str(RUNNER_DIR / "sdkconfig.defaults")] + if self.executorch: + defaults.append(str(RUNNER_DIR / "sdkconfig.executorch")) + defaults += [str(p) for p in self.extra_defaults] + args = [ + f"-DIDF_TARGET={self.target}", + f"-DSDKCONFIG={self.build_dir}/sdkconfig", + f"-DSDKCONFIG_DEFAULTS={';'.join(defaults)}", + ] + if self.executorch and self.executorch_build_dir: + args.append(f"-DESPNN_EXECUTORCH_BUILD_DIR={self.executorch_build_dir}") + if self.executorch and self.executorch_source_dir: + args.append(f"-DESPNN_EXECUTORCH_SOURCE_DIR={self.executorch_source_dir}") + return args + + def configure(self, bundle: Path, iterations: int) -> None: + self.build_dir.mkdir(parents=True, exist_ok=True) + values = { + "CONFIG_ESPNN_RUNNER_BUNDLE": f'"{bundle.resolve()}"', + "CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS": str(iterations), + } + sdkconfig = self.build_dir / "sdkconfig" + if sdkconfig.exists(): + set_sdkconfig(sdkconfig, values) + else: + overlay = self.build_dir / "sdkconfig.bundle" + overlay.write_text("".join(f"{k}={v}\n" for k, v in values.items())) + if overlay not in self.extra_defaults: + self.extra_defaults.append(overlay) + + def run_idf( + self, *actions: str, port: str | None = None, log: Path | None = None + ) -> str: + command = ( + idf_command() + + ["-C", str(RUNNER_DIR), "-B", str(self.build_dir)] + + self.cmake_args() + ) + if port: + command += ["-p", port, "-b", "921600"] + command += list(actions) + result = subprocess.run(command, capture_output=True, text=True, env=tool_env()) + output = result.stdout + result.stderr + if log: + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(output) + if result.returncode != 0: + errors = [ + line + for line in output.splitlines() + if "error" in line.lower() or "FAILED" in line + ] + errors = errors or output.strip().splitlines()[-6:] + raise RuntimeError( + f"idf.py {' '.join(actions)} failed:\n" + "\n".join(errors[-15:]) + ) + return output + + def binary_size(self) -> int: + image = self.build_dir / "espnn_runner.bin" + return image.stat().st_size if image.exists() else 0 + + +class SerialPort: + def __init__(self, path: str, baud: int = 115200): + self.fd = os.open(path, os.O_RDWR | os.O_NOCTTY) + tty.setraw(self.fd) + attributes = termios.tcgetattr(self.fd) + speed = getattr(termios, f"B{baud}") + attributes[4] = attributes[5] = speed + termios.tcsetattr(self.fd, termios.TCSANOW, attributes) + + def _lines(self, dtr: bool, rts: bool) -> None: + bits = struct.unpack( + "I", fcntl.ioctl(self.fd, termios.TIOCMGET, struct.pack("I", 0)) + )[0] + bits = (bits | TIOCM_DTR) if dtr else (bits & ~TIOCM_DTR) + bits = (bits | TIOCM_RTS) if rts else (bits & ~TIOCM_RTS) + fcntl.ioctl(self.fd, termios.TIOCMSET, struct.pack("I", bits)) + + def reset(self) -> None: + self._lines(dtr=False, rts=True) + time.sleep(0.1) + self._lines(dtr=False, rts=False) + + def read(self, timeout: float) -> bytes: + ready, _, _ = select.select([self.fd], [], [], timeout) + if not ready: + return b"" + try: + return os.read(self.fd, 65536) + except OSError: + return b"" + + def close(self) -> None: + os.close(self.fd) + + +@dataclass +class RunResult: + chip: dict = field(default_factory=dict) + cases: list[dict] = field(default_factory=list) + passed: bool = False + finished: bool = False + log: str = "" + last_error: str = "" + + +def parse_line(text: str, result: RunResult, on_case=None) -> None: + if text.startswith("CHIP "): + result.chip = dict(FIELDS.findall(text)) + elif text.startswith("ERROR:"): + result.last_error = text[len("ERROR:") :].strip() + elif text.startswith(("SUITE CASE", "ET CASE")): + case = dict(FIELDS.findall(text)) + case["kind"] = "executorch" if text.startswith("ET CASE") else "standalone" + if case.get("status") != "PASS" and result.last_error: + case["error"] = result.last_error + result.last_error = "" + result.cases.append(case) + if on_case: + on_case(case) + elif text.startswith("ESPNN RUNNER:"): + result.finished = True + result.passed = "PASS" in text + + +def capture(port: str, timeout: float, on_line=None, on_case=None) -> RunResult: + result = RunResult() + serial = SerialPort(port) + serial.reset() + deadline = time.time() + timeout + pending = b"" + raw = bytearray() + try: + while time.time() < deadline and not result.finished: + chunk = serial.read(0.5) + if not chunk: + continue + raw += chunk + pending += chunk + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + text = line.decode("utf-8", errors="replace").rstrip("\r") + parse_line(text, result, on_case) + if on_line: + on_line(text) + finally: + serial.close() + result.log = raw.decode("utf-8", errors="replace") + return result + + +def run_bundle( + build: RunnerBuild, + bundle: Path, + port: str, + iterations: int, + timeout: float, + log_dir: Path, + on_line=None, + on_case=None, + on_status=None, +) -> RunResult: + build.configure(bundle, iterations) + if on_status: + on_status("building") + build.run_idf("build", log=log_dir / "build.log") + if on_status: + on_status("flashing") + build.run_idf("flash", port=port, log=log_dir / "flash.log") + if on_status: + on_status("running on board") + result = capture(port, timeout, on_line, on_case) + (log_dir / "board.log").write_text(result.log) + return result + + +if __name__ == "__main__": + print(detect_chip(sys.argv[1] if len(sys.argv) > 1 else "/dev/ttyACM0")) diff --git a/examples/espressif/scripts/license_headers.py b/examples/espressif/scripts/license_headers.py new file mode 100644 index 00000000000..244f5641cee --- /dev/null +++ b/examples/espressif/scripts/license_headers.py @@ -0,0 +1,140 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Add or verify the BSD-3-Clause license header on every source and build file +of the ESP-NN backend and its examples. + + python examples/espressif/scripts/license_headers.py --holder "Your Name or Company" + python examples/espressif/scripts/license_headers.py --check + +The header matches the ExecuTorch convention (contributor copyright line plus +the reference to the BSD-style LICENSE at the repository root) and adds an +SPDX identifier. Re-running with a different --holder replaces the line. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SCOPES = [ROOT / "backends" / "espressif", ROOT / "examples" / "espressif"] +DEFAULT_HOLDER = "Mwaniki Njagi" +YEAR = "2026" +HASH_STYLE = {".py", ".sh", ".cmake", ".yml", ".yaml"} +HASH_NAMES = {"CMakeLists.txt", "Kconfig", "Kconfig.projbuild"} +BLOCK_STYLE = {".c", ".cc", ".cpp", ".h", ".hpp", ".S"} +SKIP_PARTS = { + "build", + "managed_components", + "__pycache__", + "benchmarks", + "bundles", + "models", +} +MARKER = "SPDX-License-Identifier: BSD-3-Clause" +BODY = [ + "This source code is licensed under the BSD-style license found in the", + "LICENSE file in the root directory of this source tree.", +] + + +def header_lines(holder: str) -> list[str]: + return [ + f"Copyright {YEAR} {holder}.", + "All rights reserved.", + "", + *BODY, + "", + MARKER, + ] + + +def render(style: str, holder: str) -> str: + lines = header_lines(holder) + if style == "hash": + return "\n".join(f"# {line}".rstrip() for line in lines) + "\n" + return "/*\n" + "\n".join(f" * {line}".rstrip() for line in lines) + "\n */\n" + + +def style_for(path: Path) -> str | None: + if path.name in HASH_NAMES or path.suffix in HASH_STYLE: + return "hash" + if path.suffix in BLOCK_STYLE: + return "block" + return None + + +def strip_existing(text: str, style: str) -> str: + if MARKER not in text.split("\n\n", 2)[0] + text[:600]: + return text + if style == "hash": + pattern = re.compile(r"\A(#[^\n]*\n)+?# " + re.escape(MARKER) + r"\n\n?") + else: + pattern = re.compile(r"\A/\*.*?" + re.escape(MARKER) + r"\n \*/\n\n?", re.S) + return pattern.sub("", text, count=1) + + +def apply(path: Path, holder: str) -> bool: + style = style_for(path) + text = path.read_text() + shebang = "" + if text.startswith("#!"): + shebang, _, text = text.partition("\n") + shebang += "\n" + body = strip_existing(text, style) + body = body.lstrip("\n") + updated = shebang + render(style, holder) + ("\n" + body if body else "") + if updated != shebang + text: + path.write_text(updated) + return True + return False + + +def files(): + for scope in SCOPES: + for path in sorted(scope.rglob("*")): + parts = path.relative_to(scope).parts[:-1] + if path.is_file() and not any( + p in SKIP_PARTS or p.startswith("build") for p in parts + ): + if style_for(path): + yield path + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--holder", default=DEFAULT_HOLDER) + parser.add_argument( + "--check", action="store_true", help="fail if any file lacks the header" + ) + args = parser.parse_args() + missing = [] + changed = 0 + for path in files(): + if args.check: + if MARKER not in path.read_text()[:800]: + missing.append(path) + elif apply(path, args.holder): + changed += 1 + if args.check: + for path in missing: + print(f"missing license header: {path.relative_to(ROOT)}") + print(f"{len(missing)} file(s) without a header") + return 1 if missing else 0 + print(f"updated {changed} file(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/espressif/scripts/make_test_bundles.py b/examples/espressif/scripts/make_test_bundles.py new file mode 100644 index 00000000000..ba05686a1b3 --- /dev/null +++ b/examples/espressif/scripts/make_test_bundles.py @@ -0,0 +1,169 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Build the on-board operator test bundles from the backend's unit-test models. + + op_suite.xtb every XTS3 opcode as a standalone int8 graph (bit-exact check) + pte_suite.xtp ExecuTorch programs: full delegation, multiple inputs and + outputs, and CPU fallback (cumsum, transformer, LSTM) + + python examples/espressif/scripts/make_test_bundles.py --output-dir examples/espressif/espnn_runner/bundles +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from collections import Counter +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from espnn_bundle import sqnr_db, write_bundle, xte1_vectors, xtv2_vectors # noqa: E402 + +from executorch.backends.espressif import to_espnn_edge # noqa: E402 +from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 +from executorch.backends.espressif.serialization import ( # noqa: E402 + build_graph, + parse_graph, + serialize_graph, +) +from executorch.backends.espressif.test import ( # noqa: E402 # noqa: E402 # noqa: E402 + test_cpu_fallback as fallback_cases, + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) +from executorch.exir.lowered_backend_module import LoweredBackendModule # noqa: E402 + + +def standalone_cases(): + yield "conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + yield "concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)] + yield "se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] + for name, model, shapes, _ in op_cases.CASES: + yield name, model, shapes + + +def program_cases(): + yield "se-block-full-delegate", lowering_cases.SEBlockModel(), [ + (1, 3, 16, 16) + ], 99.0 + yield "detection-head-2-outputs", op_cases.DetectionHead(), [(1, 3, 16, 16)], 99.0 + yield "two-inputs-two-outputs", op_cases.TwoInputTwoOutput(), [ + (1, 3, 12, 12), + (1, 6), + ], 99.0 + for name, model, shapes, _ in fallback_cases.CASES: + yield name, model, shapes, 30.0 + + +def random_inputs(shapes, count, seed): + generator = torch.Generator().manual_seed(seed) + return [[torch.randn(s, generator=generator) for s in shapes] for _ in range(count)] + + +def build_op_suite(samples: int): + entries, reports = [], [] + for name, model, shapes in standalone_cases(): + _, program = op_cases.quantize(model, shapes) + blob = serialize_graph(build_graph(program)) + graph = parse_graph(blob) + vectors = xtv2_vectors( + graph, [[x.numpy() for x in s] for s in random_inputs(shapes, samples, 7)] + ) + operations = dict(Counter(op.opcode.name for op in graph.operations)) + entries.append((name, blob, vectors)) + reports.append( + { + "name": name, + "operations": operations, + "blob_bytes": len(blob), + "inputs": [list(graph.tensors[i].shape) for i in graph.inputs], + "outputs": [list(graph.tensors[i].shape) for i in graph.outputs], + } + ) + print(f"op_suite {name:34s} {sum(operations.values()):3d} ops {operations}") + return entries, reports + + +def build_pte_suite(samples: int): + entries, reports = [], [] + for name, model, shapes, min_sqnr in program_cases(): + converted, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program) + exported = edge.exported_program() + emulator = ProgramEmulator(exported) + rows, worst = [], math.inf + for inputs in random_inputs(shapes, samples, 11): + outputs = emulator(*inputs) + outputs = list(outputs) if isinstance(outputs, (tuple, list)) else [outputs] + expected = converted(*inputs) + expected = ( + list(expected) if isinstance(expected, (tuple, list)) else [expected] + ) + worst = min( + [worst] + + [sqnr_db(w, g.to(torch.float32)) for w, g in zip(expected, outputs)] + ) + rows.append((inputs, outputs)) + exact = not report["cpu_ops"] + entries.append( + (name, edge.to_executorch().buffer, xte1_vectors(rows, exact, min_sqnr)) + ) + delegates = sum( + isinstance(m, LoweredBackendModule) + for _, m in exported.graph_module.named_children() + ) + reports.append( + { + "name": name, + "delegates": delegates, + "partitions": report["partitions"], + "delegated_ops": report["delegated_ops"], + "cpu_ops": report["cpu_ops"], + "exact_required": exact, + "min_sqnr_db": min_sqnr, + "host_sqnr_vs_pytorch_db": round(worst, 1), + "inputs": [list(t.shape) for t in rows[0][0]], + "outputs": [list(t.shape) for t in rows[0][1]], + } + ) + print( + f"pte_suite {name:34s} delegates={delegates} cpu_ops={len(report['cpu_ops'])}" + ) + return entries, reports + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output-dir", + type=Path, + default=Path(__file__).resolve().parents[1] / "espnn_runner" / "bundles", + ) + parser.add_argument("--samples", type=int, default=3) + args = parser.parse_args() + torch.set_grad_enabled(False) + for filename, magic, builder in ( + ("op_suite.xtb", b"XTB1", build_op_suite), + ("pte_suite.xtp", b"XTP1", build_pte_suite), + ): + entries, reports = builder(args.samples) + path = args.output_dir / filename + size = write_bundle(path, magic, entries) + path.with_suffix(".json").write_text(json.dumps(reports, indent=2)) + print(f"wrote {path} ({size} bytes, {len(entries)} cases)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/scripts/run_bundle.py b/examples/espressif/scripts/run_bundle.py new file mode 100644 index 00000000000..0eb9ae0c878 --- /dev/null +++ b/examples/espressif/scripts/run_bundle.py @@ -0,0 +1,79 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Build examples/espressif/espnn_runner for the connected chip, flash a bundle +(.xtb int8 graphs or .xtp ExecuTorch programs) and report each case. + + python examples/espressif/scripts/run_bundle.py --port /dev/ttyUSB0 \ + examples/espressif/espnn_runner/bundles/op_suite.xtb +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import espnn_board as board # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("bundle", type=Path) + parser.add_argument("--port", default="/dev/ttyACM0") + parser.add_argument("--target", help="override the detected chip") + parser.add_argument("--iterations", type=int, default=10) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--build-root", type=Path, default=board.RUNNER_DIR / "build") + parser.add_argument("--executorch-build-dir", type=Path) + parser.add_argument("--executorch-source-dir", type=Path) + parser.add_argument("--json", type=Path, help="write the parsed results here") + args = parser.parse_args() + + chip = board.detect_chip(args.port) + target = args.target or chip.target + executorch = args.bundle.read_bytes()[:4] == b"XTP1" + mode = "executorch" if executorch else "standalone" + print( + f"{chip.description}: {chip.flash_mb} MB flash; building {mode} runner for {target}", + flush=True, + ) + build = board.RunnerBuild( + target=target, + executorch=executorch, + build_dir=args.build_root / f"{target}-{mode}", + executorch_build_dir=args.executorch_build_dir, + executorch_source_dir=args.executorch_source_dir, + ) + result = board.run_bundle( + build, + args.bundle, + args.port, + args.iterations, + args.timeout, + build.build_dir / "logs", + on_line=lambda line: print(line, flush=True), + ) + if args.json: + args.json.write_text( + json.dumps( + {"chip": result.chip, "cases": result.cases, "passed": result.passed}, + indent=2, + ) + ) + print(f"RESULT: {'PASS' if result.passed else 'FAIL'} ({len(result.cases)} cases)") + return 0 if result.passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 71aaa9cf9fbe7e64a51c17c758271adb2119cdff Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Sun, 18 Oct 2026 21:57:50 +0200 Subject: [PATCH 02/10] Use a relative stream URL in the camera viewer page --- examples/espressif/camera_stream/viewer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/espressif/camera_stream/viewer.py b/examples/espressif/camera_stream/viewer.py index 4996151aed3..baf435a1d2b 100644 --- a/examples/espressif/camera_stream/viewer.py +++ b/examples/espressif/camera_stream/viewer.py @@ -396,7 +396,7 @@ def reader_thread( # noqa: C901

Camera preview

-
camera stream
+
camera stream
waiting for first result
@@ -470,7 +470,7 @@ def reader_thread( # noqa: C901 const live = st.result_age_ms !== null && st.result_age_ms < 5000; $("dot").style.background = live ? "var(--good)" : "var(--warn)"; $("status-text").textContent = st.link && st.link !== "connected" ? st.link : (live ? "streaming" : (r ? "stalled" : "waiting for board")); - if (live && camStale) { camStale = false; $("cam").src = "/stream.mjpg?t=" + Date.now(); } + if (live && camStale) { camStale = false; $("cam").src = "stream.mjpg?t=" + Date.now(); } if (r) { $("m-infer").textContent = fmt(r.infer_ms, 0); $("m-pre").textContent = fmt(r.pre_ms, 0); From 566d27569580a9a46db62da0714a33e3d36a0857 Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Sun, 18 Oct 2026 22:00:13 +0200 Subject: [PATCH 03/10] Sort using-declarations in pte_suite.cpp for clang-format --- examples/espressif/espnn_runner/main/pte_suite.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/espressif/espnn_runner/main/pte_suite.cpp b/examples/espressif/espnn_runner/main/pte_suite.cpp index 58cb7cb32af..cf010387464 100644 --- a/examples/espressif/espnn_runner/main/pte_suite.cpp +++ b/examples/espressif/espnn_runner/main/pte_suite.cpp @@ -45,8 +45,8 @@ namespace executorch { namespace examples { namespace espressif { -using ::executorch::backends::espressif::DelegateStats; using ::executorch::backends::espressif::delegate_stats; +using ::executorch::backends::espressif::DelegateStats; using ::executorch::backends::espressif::read_le_u32; using ::executorch::backends::espressif::register_espnn_backend; using ::executorch::backends::espressif::reset_delegate_stats; From 4b4c5c719228a26b7ac8e89726a8b3b926de3aab Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Sun, 18 Oct 2026 22:50:58 +0200 Subject: [PATCH 04/10] Address review: parser overflows, strict mode, shared PAL, CI coverage Harden the XTS3 parser against COPY/UPSAMPLE size overflows and misaligned int32 constants; enforce allow_cpu_fallback=False; drop the component PAL in favour of executor_runner/esp_pal.cpp; serialize invoke(); check delegate I/O shape and dim order; enforce runtime limits at serialization; honour per-module quantizer configs; add backends/espressif/test to pytest.ini and run the operator suite in esp-emu in the Espressif CI job. --- .ci/scripts/test_espressif.sh | 19 ++ .github/workflows/test-espressif.yml | 1 + backends/espressif/README.md | 6 +- backends/espressif/espnn_preprocess.py | 21 ++- .../espressif/espnn_runtime/CMakeLists.txt | 3 - backends/espressif/espnn_runtime/Kconfig | 5 - .../include/espnn/int8_interpreter.h | 8 +- .../espnn_runtime/include/espnn/xts3_graph.h | 2 + .../espnn_runtime/src/espnn_backend.cpp | 23 ++- .../espressif/espnn_runtime/src/espnn_pal.cpp | 59 ------ .../espnn_runtime/src/int8_interpreter.cpp | 23 +++ .../espnn_runtime/src/xts3_graph.cpp | 22 ++- .../espressif/partition/espnn_partitioner.py | 38 +++- .../espressif/quantizer/espnn_quantizer.py | 83 +++++++-- backends/espressif/serialization/xts3.py | 55 +++++- .../espressif/test/host/xts3_parse_main.cpp | 37 ++++ backends/espressif/test/test_contracts.py | 173 ++++++++++++++++++ backends/espressif/test/test_cpu_fallback.py | 3 +- .../espressif/test/test_fold_batch_norm.py | 4 +- backends/espressif/test/test_int8_lowering.py | 8 +- backends/espressif/test/test_int8_ops.py | 26 ++- .../espressif/test/test_xts3_parser_host.py | 170 +++++++++++++++++ examples/espressif/camera_stream/flash.sh | 2 +- examples/espressif/camera_stream/viewer.py | 13 +- .../espnn_runner/main/CMakeLists.txt | 4 +- .../espressif/espnn_runner/sdkconfig.emulator | 1 + examples/espressif/scripts/benchmark.py | 10 +- examples/espressif/scripts/espnn_board.py | 5 +- .../espressif/scripts/make_test_bundles.py | 15 +- examples/espressif/scripts/run_bundle.py | 8 +- pytest.ini | 1 + 31 files changed, 711 insertions(+), 137 deletions(-) delete mode 100644 backends/espressif/espnn_runtime/src/espnn_pal.cpp create mode 100644 backends/espressif/test/host/xts3_parse_main.cpp create mode 100644 backends/espressif/test/test_contracts.py create mode 100644 backends/espressif/test/test_xts3_parser_host.py create mode 100644 examples/espressif/espnn_runner/sdkconfig.emulator diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index ca59bba695d..ac630a2a967 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -66,3 +66,22 @@ timeout 130s esp-emu --chip esp32s3 \ --net user --psram-size 8M --timeout 120s --exit-on 'Program complete.' \ 2>&1 | tee "${build_dir}/emulator.log" python examples/espressif/test_emulator.py "${build_dir}/emulator.log" + +# ESP-NN backend: run the standalone operator suite and compare every case +# bit-exactly with the host reference. The emulator does not implement the +# ESP32-S3 SIMD instructions, so ESP-NN is built with its ANSI C kernels. +python examples/espressif/scripts/make_test_bundles.py --suites op \ + --output-dir "${build_dir}/espnn-bundles" 2>&1 | tee "${build_dir}/espnn-bundles.log" +printf 'CONFIG_ESPNN_RUNNER_BUNDLE="%s"\n' "${build_dir}/espnn-bundles/op_suite.xtb" \ + > "${build_dir}/espnn-bundle.defaults" +idf.py -C examples/espressif/espnn_runner -B "${build_dir}/espnn" \ + -DIDF_TARGET=esp32s3 \ + -DSDKCONFIG="${build_dir}/espnn-sdkconfig" \ + "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;sdkconfig.emulator;${build_dir}/espnn-bundle.defaults" \ + build merge-bin 2>&1 | tee "${build_dir}/espnn-build.log" +timeout 330s esp-emu --chip esp32s3 \ + --firmware "${build_dir}/espnn/merged-binary.bin" \ + --elf "${build_dir}/espnn/espnn_runner.elf" \ + --net user --psram-size 8M --timeout 300s --exit-on 'ESPNN RUNNER:' \ + 2>&1 | tee "${build_dir}/espnn-emulator.log" +grep -q "ESPNN RUNNER: PASS" "${build_dir}/espnn-emulator.log" diff --git a/.github/workflows/test-espressif.yml b/.github/workflows/test-espressif.yml index ddba1f2960e..5503b3845fc 100644 --- a/.github/workflows/test-espressif.yml +++ b/.github/workflows/test-espressif.yml @@ -7,6 +7,7 @@ on: - release/* pull_request: paths: + - backends/espressif/** - examples/espressif/** - tools/cmake/preset/esp_baremetal.cmake - CMakePresets.json diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 5417bb5d169..543df9f4ba5 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -171,11 +171,15 @@ menuconfig: *ExecuTorch ESP-NN backend*. The component links `cmake-out-esp-${IDF_TARGET}`. It also registers `EspNNBackend` with the runtime at startup. +The component does not define the ExecuTorch platform layer (`et_pal_*`). +Applications either compile the shared +[`examples/espressif/executor_runner/esp_pal.cpp`](../../examples/espressif/executor_runner/esp_pal.cpp), +as `espnn_runner` does, or provide their own. + Other options: | Kconfig | Default | Effect | |---|---|---| -| `ESPNN_PROVIDE_EXECUTORCH_PAL` | y | ExecuTorch platform layer: `esp_timer` clock, `printf` logs, PSRAM-aware allocator | | `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | | `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | | `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py index 184d9a422d1..1b7f7f32376 100644 --- a/backends/espressif/espnn_preprocess.py +++ b/backends/espressif/espnn_preprocess.py @@ -9,8 +9,6 @@ from __future__ import annotations import logging -import os -from pathlib import Path from typing import List from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult @@ -22,7 +20,6 @@ BACKEND_ID = "EspNNBackend" INT8_FORMAT_SPEC = CompileSpec("espnn_format", b"int8") -DUMP_ENV = "ESPNN_DELEGATE_OUTPUT" class EspNNBackend(BackendDetails): @@ -32,14 +29,23 @@ class EspNNBackend(BackendDetails): Each partition produced by EspNNPartitioner is lowered to an XTS3 int8 graph: ESP-NN operator records, per-channel requantization parameters, 16-byte aligned int8/int32 constants and an input/output table. The - on-device EspNNBackend (backends/espressif/runtime) parses the blob and - executes it with ESP-NN kernels. + on-device EspNNBackend (backends/espressif/espnn_runtime) parses the blob + and executes it with ESP-NN kernels. - Set ESPNN_DELEGATE_OUTPUT= to also write the last blob to a file. + Compile specs: `espnn_format` selects the payload format; only `int8` is + defined, and any other value is rejected. """ @staticmethod def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResult: + for spec in compile_specs: + if ( + spec.key == INT8_FORMAT_SPEC.key + and spec.value != INT8_FORMAT_SPEC.value + ): + raise ValueError( + f"Unsupported espnn_format {spec.value!r}; only b'int8' is defined" + ) graph = build_graph(edge_program) blob = serialize_graph(graph) logger.info( @@ -49,9 +55,6 @@ def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResu len(graph.constant_data), len(blob), ) - output_path = os.environ.get(DUMP_ENV) - if output_path: - Path(output_path).write_bytes(blob) return PreprocessResult(processed_bytes=blob, debug_handle_map={}) diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt index c14c90004df..a0209a7be5f 100644 --- a/backends/espressif/espnn_runtime/CMakeLists.txt +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -9,9 +9,6 @@ set(espnn_sources "src/xts3_graph.cpp" "src/int8_interpreter.cpp") if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) list(APPEND espnn_sources "src/espnn_backend.cpp") - if(CONFIG_ESPNN_PROVIDE_EXECUTORCH_PAL) - list(APPEND espnn_sources "src/espnn_pal.cpp") - endif() endif() idf_component_register( diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index 28152ff3760..c9cc88b65f1 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -16,11 +16,6 @@ menu "ExecuTorch ESP-NN backend" examples/espressif/scripts/build_executorch_esp.sh. Without it the component still provides the standalone XTS3 int8 interpreter. - config ESPNN_PROVIDE_EXECUTORCH_PAL - bool "Provide the ExecuTorch platform layer (esp_timer clock, printf logs, PSRAM allocator)" - default y - depends on ESPNN_EXECUTORCH_DELEGATE - config ESPNN_DELEGATE_CONSTANTS_TO_PSRAM bool "Copy delegate weights from flash into PSRAM when they fit" default y diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h index 11bcc22d9f5..9c18ac8e63b 100644 --- a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -35,7 +35,9 @@ struct Int8RuntimeOptions { bool copy_constants_to_psram = false; /// Place the activation arena in internal RAM when it fits, else PSRAM. bool prefer_internal_arena = true; - /// Split large convolutions across both cores. + /// Split large convolutions across both cores. ESP-NN's worker task is + /// process-wide, so once any interpreter enables it every interpreter's + /// kernels may use it; results are bit-identical either way. bool dual_core = false; /// Internal RAM left free when deciding whether the arena fits there. size_t internal_reserve_bytes = 48 * 1024; @@ -93,6 +95,10 @@ class Int8Interpreter { /** * Runs every operation once on the current input data. * + * ESP-NN keeps its scratch buffers in global state, so invoke() calls from + * all interpreters are serialized by one process-wide mutex. Concurrent + * callers on other tasks or cores wait rather than share buffers. + * * @retval false The graph is not loaded or an operation failed. */ bool invoke(); diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h index cc74285f91c..4fac0173531 100644 --- a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -64,6 +64,8 @@ static constexpr size_t UPSAMPLE_HEADER_SIZE = 20; static constexpr int32_t UPSAMPLE_MODE_NEAREST = 0; static constexpr int32_t UPSAMPLE_MODE_BILINEAR = 1; static constexpr int32_t UPSAMPLE_WEIGHT_BITS = 10; +/// Largest output height or width an UPSAMPLE operation may declare. +static constexpr int32_t UPSAMPLE_MAX_EXTENT = 65535; static constexpr size_t TRANSPOSE_CONV_PARAMS_SIZE = 52; /// Operation types; values are part of the file format. diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 4e542706c5e..7d247f6d141 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -65,8 +66,16 @@ int8_t quantize(float value, float inverse_scale, int32_t zero_point) { return static_cast(q); } -bool same_elements(const Tensor& tensor, const Int8TensorInfo& info) { - return static_cast(tensor.numel()) == info.bytes; +bool same_layout(const Tensor& tensor, const Int8TensorInfo& info) { + if (tensor.dim() != info.rank || + static_cast(tensor.numel()) != info.bytes) + return false; + for (uint8_t d = 0; d < info.rank; ++d) { + if (tensor.size(d) != info.shape[d]) + return false; + } + return executorch::runtime::is_contiguous_dim_order( + tensor.dim_order().data(), tensor.dim_order().size()); } bool load_input( @@ -74,7 +83,7 @@ bool load_input( size_t index, const Tensor& tensor) { Int8TensorInfo info{}; - if (!interpreter.input_info(index, info) || !same_elements(tensor, info)) + if (!interpreter.input_info(index, info) || !same_layout(tensor, info)) return false; int8_t* destination = interpreter.input_data(index); const size_t plane = static_cast(info.height) * info.width; @@ -117,7 +126,7 @@ bool store_output( size_t index, Tensor& tensor) { Int8TensorInfo info{}; - if (!interpreter.output_info(index, info) || !same_elements(tensor, info)) + if (!interpreter.output_info(index, info) || !same_layout(tensor, info)) return false; const int8_t* source = interpreter.output_data(index); const size_t plane = static_cast(info.height) * info.width; @@ -190,6 +199,8 @@ class EspNNBackendInterface final : public BackendInterface { state->~DelegateState(); return Error::DelegateInvalidCompatibility; } + if (state->interpreter.constants_copied()) + processed->Free(); ++g_stats.delegates; g_stats.arena_bytes += state->interpreter.arena_bytes(); return state; @@ -215,7 +226,7 @@ class EspNNBackendInterface final : public BackendInterface { if (!args[i]->isTensor() || !load_input(interpreter, i, args[i]->toTensor())) { std::printf( - "EspNN delegate: input %u has an unsupported dtype or size\n", + "EspNN delegate: input %u has an unsupported dtype, shape or dim order\n", static_cast(i)); return Error::InvalidArgument; } @@ -228,7 +239,7 @@ class EspNNBackendInterface final : public BackendInterface { Tensor tensor = args[inputs + o]->toTensor(); if (!store_output(interpreter, o, tensor)) { std::printf( - "EspNN delegate: output %u has an unsupported dtype or size\n", + "EspNN delegate: output %u has an unsupported dtype, shape or dim order\n", static_cast(o)); return Error::InvalidArgument; } diff --git a/backends/espressif/espnn_runtime/src/espnn_pal.cpp b/backends/espressif/espnn_runtime/src/espnn_pal.cpp deleted file mode 100644 index 2926893d95a..00000000000 --- a/backends/espressif/espnn_runtime/src/espnn_pal.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include -#include - -#include - -#include -#include - -extern "C" { - -void et_pal_init(void) {} - -ET_NORETURN void et_pal_abort(void) { - std::printf("ExecuTorch abort\n"); - std::abort(); -} - -et_timestamp_t et_pal_current_ticks(void) { - return static_cast(esp_timer_get_time()); -} - -et_tick_ratio_t et_pal_ticks_to_ns_multiplier(void) { - return {1000, 1}; -} - -void et_pal_emit_log_message( - et_timestamp_t, - et_pal_log_level_t level, - const char* filename, - const char*, - size_t line, - const char* message, - size_t) { - std::printf( - "ET %c %s:%u %s\n", - static_cast(level), - filename, - static_cast(line), - message); -} - -void* et_pal_allocate(size_t size) { - return heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); -} - -void et_pal_free(void* pointer) { - heap_caps_free(pointer); -} -} diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp index c6d1848eee7..391574781c2 100644 --- a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace executorch { @@ -30,6 +31,27 @@ namespace { constexpr size_t ARENA_ALIGNMENT = 16; constexpr int64_t YIELD_INTERVAL_US = 1000000; +SemaphoreHandle_t invoke_mutex() { + static StaticSemaphore_t storage; + static SemaphoreHandle_t handle = xSemaphoreCreateMutexStatic(&storage); + return handle; +} + +class InvokeLock { + public: + InvokeLock() : handle_(invoke_mutex()) { + xSemaphoreTake(handle_, portMAX_DELAY); + } + ~InvokeLock() { + xSemaphoreGive(handle_); + } + InvokeLock(const InvokeLock&) = delete; + InvokeLock& operator=(const InvokeLock&) = delete; + + private: + SemaphoreHandle_t handle_; +}; + size_t align_up(size_t value, size_t alignment) { return (value + alignment - 1) & ~(alignment - 1); } @@ -1652,6 +1674,7 @@ void Int8Interpreter::run_transposed_conv( bool Int8Interpreter::invoke() { if (!loaded_) return false; + InvokeLock lock; if (scratch_ != nullptr) { esp_nn_set_conv_scratch_buf(scratch_); esp_nn_set_depthwise_conv_scratch_buf(scratch_); diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp index 03674f98a81..af6b6c38453 100644 --- a/backends/espressif/espnn_runtime/src/xts3_graph.cpp +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -172,18 +172,23 @@ bool copy_extent_valid( const int32_t* strides, int32_t offset, size_t limit) { + if (limit == 0 || limit > static_cast(INT32_MAX)) + return false; + const int64_t bound = static_cast(limit); int64_t low = offset; int64_t high = offset; for (int32_t d = 0; d < rank; ++d) { if (dims[d] <= 0) return false; const int64_t span = static_cast(dims[d] - 1) * strides[d]; + if (span <= -bound || span >= bound) + return false; if (span < 0) low += span; else high += span; } - return low >= 0 && high < static_cast(limit); + return low >= 0 && high < bound; } bool validate_copy(const GraphView& graph, const OperationRecord& op) { @@ -198,6 +203,8 @@ bool validate_copy(const GraphView& graph, const OperationRecord& op) { const int32_t fill = read_i32(p + 8); if (segments <= 0 || (flags & ~COPY_FLAG_FILL) != 0 || fill < -128 || fill > 127 || + static_cast(segments) > + (op.parameter_size - COPY_HEADER_SIZE) / COPY_SEGMENT_SIZE || op.parameter_size != COPY_HEADER_SIZE + static_cast(segments) * COPY_SEGMENT_SIZE) { @@ -266,9 +273,11 @@ bool validate_upsample(const GraphView& graph, const OperationRecord& op) { const int32_t out_w = read_i32(p + 16); if ((mode != UPSAMPLE_MODE_NEAREST && mode != UPSAMPLE_MODE_BILINEAR) || in_h != a.shape[2] || in_w != a.shape[3] || out_h != out.shape[2] || - out_w != out.shape[3] || - op.parameter_size != - UPSAMPLE_HEADER_SIZE + 12U * static_cast(out_h + out_w)) { + out_w != out.shape[3] || out_h <= 0 || out_w <= 0 || + out_h > UPSAMPLE_MAX_EXTENT || out_w > UPSAMPLE_MAX_EXTENT || + static_cast(op.parameter_size) != + UPSAMPLE_HEADER_SIZE + + 12ULL * (static_cast(out_h) + out_w)) { return false; } const int32_t one = 1 << UPSAMPLE_WEIGHT_BITS; @@ -944,7 +953,12 @@ bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { return false; } const size_t data_size = tensor_data_size(tensor); + const bool misaligned_int32 = is_constant && + tensor.dtype == static_cast(DataType::INT32) && + (tensor.constant_offset % sizeof(int32_t)) != 0; if (data_size == 0 || + tensor_element_count(tensor) > static_cast(INT32_MAX) || + misaligned_int32 || (is_constant && !range_valid( tensor.constant_offset, data_size, graph.header.constant_size)) || diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index de9380c6e59..6b4bf9a6cd1 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -44,6 +44,7 @@ from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner from torch.fx.passes.infra.pass_base import PassBase, PassResult from torch.fx.passes.operator_support import OperatorSupportBase +from torch.fx.passes.utils.fuser_utils import validate_partition SUPPORTED_KINDS = { @@ -276,7 +277,9 @@ def _attach_outputs_to_producers(partitions: list[set]) -> None: """ A quantize node (and an activation fused into its producer) must live in the producer's partition so the partition exports int8 or dequantized - values only, never a raw accumulator. + values only, never a raw accumulator. A move that would make either + partition non-convex is skipped; the affected partition then fails + _boundary_is_valid and stays on the CPU instead of breaking fusion. """ owner = {} for index, nodes in enumerate(partitions): @@ -297,7 +300,12 @@ def _attach_outputs_to_producers(partitions: list[set]) -> None: ) if not attach or owner.get(user) == index or user not in owner: continue - partitions[owner[user]].discard(user) + donor = partitions[owner[user]] + if not validate_partition( + list(nodes | {user}) + ) or not validate_partition(list(donor - {user})): + continue + donor.discard(user) nodes.add(user) owner[user] = index changed = True @@ -314,8 +322,9 @@ class EspNNPartitioner(Partitioner): With allow_cpu_fallback (the default) operators the int8 backend cannot lower stay in the ExecuTorch program and run on the CPU through the portable kernels; partition boundaries carry float or int8 tensors. - With allow_cpu_fallback=False any unsupported operator is an error, which - is what the standalone XTS3 firmware path requires. + With allow_cpu_fallback=False any operator left outside a delegate is an + error, whether it is unsupported or its region could not be delegated; + the standalone XTS3 firmware path requires this. """ def __init__( @@ -378,10 +387,6 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: for node in nodes: node.meta["delegation_tag"] = tag tags[tag] = self.delegation_spec - if not tags and not self.allow_cpu_fallback: - raise RuntimeError("No ESP32 int8 partition could be formed") - tag_constant_data(exported_program) - delegated = { node for node in graph_module.graph.nodes @@ -394,6 +399,20 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: and node not in delegated and not _is_passthrough(node) ] + leftover = [node for node in cpu_ops if "getitem" not in str(node.target)] + if leftover and not self.allow_cpu_fallback: + raise RuntimeError( + f"{len(leftover)} operator(s) would run on the CPU " + f"({rejected} region(s) could not be delegated): " + + ", ".join( + f"{node.name} ({_operator_name(node.target)}): " + f"{reasons.get(node, 'outside the delegated regions')}" + for node in leftover[:20] + ) + ) + if not tags and not self.allow_cpu_fallback: + raise RuntimeError("No ESP32 int8 partition could be formed") + tag_constant_data(exported_program) self.report = { "partitions": len(tags), "rejected_partitions": rejected, @@ -404,8 +423,7 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: "target": _operator_name(node.target), "reason": reasons.get(node, "outside the delegated regions"), } - for node in cpu_ops - if "getitem" not in str(node.target) + for node in leftover ], } return PartitionResult( diff --git a/backends/espressif/quantizer/espnn_quantizer.py b/backends/espressif/quantizer/espnn_quantizer.py index 0b710faf3f9..f7cb497119a 100644 --- a/backends/espressif/quantizer/espnn_quantizer.py +++ b/backends/espressif/quantizer/espnn_quantizer.py @@ -8,11 +8,12 @@ from __future__ import annotations -from typing import Optional +from typing import Callable, Optional, Union import torch from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( + _get_module_type_filter, get_symmetric_quantization_config, XNNPACKQuantizer, ) @@ -24,6 +25,7 @@ FixedQParamsQuantizationSpec, get_bias_qspec, get_input_act_qspec, + get_module_name_filter, get_output_act_qspec, get_weight_qspec, QuantizationAnnotation, @@ -69,8 +71,12 @@ def fused_activation_kind(node: Node) -> Optional[str]: if node.target in RELU6_TARGETS: return "relu6" if node.target in HARDTANH_TARGETS: - minimum = float(node.args[1]) if len(node.args) > 1 else -1.0 - maximum = float(node.args[2]) if len(node.args) > 2 else 1.0 + minimum = float( + node.args[1] if len(node.args) > 1 else node.kwargs.get("min_val", -1.0) + ) + maximum = float( + node.args[2] if len(node.args) > 2 else node.kwargs.get("max_val", 1.0) + ) if minimum == 0.0 and maximum == 6.0: return "relu6" return None @@ -150,10 +156,29 @@ def _annotate_add_activation( return True +ConfigSource = Union[ + QuantizationConfig, Callable[[Node], Optional[QuantizationConfig]], None +] + + +def _config_lookup( + config: ConfigSource, +) -> Callable[[Node], Optional[QuantizationConfig]]: + if callable(config) and not isinstance(config, QuantizationConfig): + return config + return lambda node: config + + def annotate_fused_activations( model: torch.fx.GraphModule, - config: QuantizationConfig, + config: ConfigSource, ) -> int: + """ + Annotate conv/linear/add followed by ReLU or ReLU6 as one fused unit. + `config` is a QuantizationConfig or a per-node lookup returning one, or + None for nodes that must stay in float. + """ + config_for = _config_lookup(config) fused = 0 for node in model.graph.nodes: if fused_activation_kind(node) is None: @@ -163,6 +188,9 @@ def annotate_fused_activations( continue if len(producer.users) != 1: continue + config = config_for(node) + if config is None or config_for(producer) is not config: + continue if producer.target in CONV_TARGETS or producer.target in LINEAR_TARGETS: fused += int(_annotate_weighted_activation(producer, node, config)) elif producer.target in ADD_TARGETS: @@ -189,6 +217,21 @@ def __init__(self, per_channel: bool = True, is_qat: bool = False): get_symmetric_quantization_config(is_per_channel=per_channel, is_qat=is_qat) ) self.fused_activation_count = 0 + self.rewritten_split_count = 0 + + def node_config(self, node: Node) -> Optional[QuantizationConfig]: + """ + The config that governs `node`, with the same precedence as + XNNPACKQuantizer: module name, then module type, then global. + None means the node stays in float. + """ + for name, config in self.module_name_config.items(): + if get_module_name_filter(name)(node): + return config + for module_type, config in self.module_type_config.items(): + if _get_module_type_filter(module_type)(node): + return config + return self.global_config def transform_for_annotation( self, model: torch.fx.GraphModule @@ -198,20 +241,19 @@ def transform_for_annotation( return model def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: - if self.global_config is not None: - self.fused_activation_count = annotate_fused_activations( - model, self.global_config - ) + self.fused_activation_count = annotate_fused_activations( + model, self.node_config + ) model = self._annotate_for_quantization_config(model) - if self.global_config is not None: - annotate_unary_and_softmax(model, self.global_config) - propagate_shared_quantization(model) + annotate_unary_and_softmax(model, self.node_config) + propagate_shared_quantization(model, self.node_config) return model def annotate_unary_and_softmax( - model: torch.fx.GraphModule, config: QuantizationConfig + model: torch.fx.GraphModule, config: ConfigSource ) -> None: + config_for = _config_lookup(config) softmax_output = FixedQParamsQuantizationSpec( dtype=torch.int8, scale=SOFTMAX_OUTPUT_SCALE, @@ -226,7 +268,8 @@ def annotate_unary_and_softmax( if node.target not in LUT_TARGETS and node.target not in SOFTMAX_TARGETS: continue source = node.args[0] - if not isinstance(source, Node): + config = config_for(node) + if not isinstance(source, Node) or config is None: continue node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( input_qspec_map={source: get_input_act_qspec(config)}, @@ -333,6 +376,9 @@ def rewrite_splits_as_slices(model: torch.fx.GraphModule) -> int: return rewritten +_ALWAYS = object() + + def _shares_qparams(node: Node) -> bool: return node.op == "call_function" and ( _is_share_obs_or_fq_op(node.target) @@ -340,11 +386,18 @@ def _shares_qparams(node: Node) -> bool: ) -def propagate_shared_quantization(model: torch.fx.GraphModule) -> None: +def propagate_shared_quantization( + model: torch.fx.GraphModule, config: ConfigSource = _ALWAYS +) -> None: + """ + Give shape and movement ops their input's quantization. With a per-node + `config` lookup, nodes whose config is None are left in float. + """ + config_for = _config_lookup(config) for node in model.graph.nodes: if not _shares_qparams(node): continue - if _is_annotated(node): + if _is_annotated(node) or config_for(node) is None: continue source = node.args[0] if not isinstance(source, Node): diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index b3f7a07d41e..b9cc0fdbc28 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -52,6 +52,10 @@ UPSAMPLE_WEIGHT_BITS = 10 UPSAMPLE_WEIGHT_ONE = 1 << UPSAMPLE_WEIGHT_BITS TRANSPOSE_CONV_PARAMS = "<13i" +UPSAMPLE_MAX_EXTENT = 65535 +MAX_GRAPH_IO = 16 +MAX_GRAPH_TENSORS = 1024 +MAX_GRAPH_OPERATIONS = 1024 TRANSPOSE_CONV_PARAMS_SIZE = struct.calcsize(TRANSPOSE_CONV_PARAMS) HEADER_FORMAT = "<4sHIIII" @@ -1866,10 +1870,56 @@ def _serialize_operation(operation: XTS3Operation, params_offset: int) -> bytes: ) -def serialize_graph(graph: XTS3Graph) -> bytes: - """Serialize to the XTS3 wire format; constants start 16-byte aligned.""" +def check_runtime_limits(graph: XTS3Graph) -> None: + """ + Raise ValueError for graphs the device runtime would reject at init(): + the same limits espnn_runtime/src/xts3_graph.cpp enforces. + """ if graph.version not in XTS3_VERSIONS: raise ValueError(f"Cannot serialize XTS3 version {graph.version}") + if len(graph.inputs) > MAX_GRAPH_IO or len(graph.outputs) > MAX_GRAPH_IO: + raise ValueError( + f"Partition has {len(graph.inputs)} inputs and {len(graph.outputs)} " + f"outputs; the ESP-NN runtime supports at most {MAX_GRAPH_IO} of each" + ) + if len(graph.tensors) > MAX_GRAPH_TENSORS: + raise ValueError( + f"Partition has {len(graph.tensors)} tensors; the ESP-NN runtime " + f"supports at most {MAX_GRAPH_TENSORS}" + ) + if len(graph.operations) > MAX_GRAPH_OPERATIONS: + raise ValueError( + f"Partition has {len(graph.operations)} operations; the ESP-NN " + f"runtime supports at most {MAX_GRAPH_OPERATIONS}" + ) + newest = XTS3OpCode.SOFTMAX if graph.version == 5 else max(XTS3OpCode) + for operation in graph.operations: + if int(operation.opcode) > int(newest): + raise ValueError( + f"{XTS3OpCode(operation.opcode).name} needs XTS3 version 6, " + f"the graph is version {graph.version}" + ) + if operation.opcode == XTS3OpCode.UPSAMPLE: + _, _, _, out_h, out_w = struct.unpack_from( + UPSAMPLE_HEADER, operation.params + ) + if max(out_h, out_w) > UPSAMPLE_MAX_EXTENT: + raise ValueError( + f"Upsample output {out_h}x{out_w} exceeds " + f"{UPSAMPLE_MAX_EXTENT} in one dimension" + ) + for tensor in graph.tensors: + if ( + tensor.flags & TENSOR_FLAG_CONSTANT + and tensor.dtype == XTS3DType.INT32 + and tensor.data_offset % 4 + ): + raise ValueError(f"int32 constant {tensor.tensor_id} is not 4-byte aligned") + + +def serialize_graph(graph: XTS3Graph) -> bytes: + """Serialize to the XTS3 wire format; constants start 16-byte aligned.""" + check_runtime_limits(graph) tensor_data = b"".join(_serialize_tensor(tensor) for tensor in graph.tensors) parameter_data = bytearray() operation_data = bytearray() @@ -2034,6 +2084,7 @@ def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 __all__ = [ "MAGIC", + "check_runtime_limits", "XTS3_VERSION", "XTS3_VERSIONS", "XTS3DType", diff --git a/backends/espressif/test/host/xts3_parse_main.cpp b/backends/espressif/test/host/xts3_parse_main.cpp new file mode 100644 index 00000000000..9fb02db1b7f --- /dev/null +++ b/backends/espressif/test/host/xts3_parse_main.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +int main(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::FILE* file = std::fopen(argv[i], "rb"); + if (file == nullptr) { + std::printf("%s missing\n", argv[i]); + return 2; + } + std::vector data; + uint8_t chunk[4096]; + size_t read = 0; + while ((read = std::fread(chunk, 1, sizeof(chunk), file)) > 0) { + data.insert(data.end(), chunk, chunk + read); + } + std::fclose(file); + executorch::backends::espressif::GraphView graph{}; + const bool accepted = executorch::backends::espressif::parse_graph( + data.data(), data.size(), graph); + std::printf("VERDICT %s %s\n", argv[i], accepted ? "accept" : "reject"); + } + return 0; +} diff --git a/backends/espressif/test/test_contracts.py b/backends/espressif/test/test_contracts.py new file mode 100644 index 00000000000..07b35efc99f --- /dev/null +++ b/backends/espressif/test/test_contracts.py @@ -0,0 +1,173 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Contract tests for the partitioner, quantizer, preprocess and serializer: +strict delegation, per-module quantization configs, and the device runtime +limits that must be caught on the host. + + pytest backends/espressif/test/test_contracts.py +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +pytest.importorskip("torchao") + +import torch # noqa: E402 +import torch.nn as nn # noqa: E402 +from executorch.backends.espressif.espnn_preprocess import ( # noqa: E402 + EspNNBackend, + INT8_FORMAT_SPEC, +) +from executorch.backends.espressif.lowering import to_espnn_edge # noqa: E402 +from executorch.backends.espressif.partition import espnn_partitioner # noqa: E402 +from executorch.backends.espressif.quantizer.espnn_quantizer import ( # noqa: E402 + EspNNQuantizer, + fused_activation_kind, +) +from executorch.backends.espressif.serialization.xts3 import ( # noqa: E402 + build_graph, + check_runtime_limits, + MAX_GRAPH_IO, + MAX_GRAPH_OPERATIONS, + TENSOR_FLAG_CONSTANT, + XTS3DType, + XTS3OpCode, +) +from executorch.backends.espressif.test.test_int8_ops import quantize # noqa: E402 +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( # noqa: E402 + get_symmetric_quantization_config, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec # noqa: E402 +from torchao.quantization.pt2e.quantize_pt2e import prepare_pt2e # noqa: E402 + + +class FeaturesAndClassifier(nn.Module): + def __init__(self): + super().__init__() + self.features = nn.Sequential(nn.Conv2d(3, 8, 3, padding=1), nn.ReLU()) + self.classifier = nn.Sequential(nn.Conv2d(8, 4, 1), nn.ReLU()) + + def forward(self, x): + return self.classifier(self.features(x)) + + +def _small_graph(): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + return build_graph(program) + + +def test_strict_mode_raises_when_a_region_stays_on_cpu(monkeypatch): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + monkeypatch.setattr( + espnn_partitioner, "_boundary_is_valid", lambda nodes, resolver: False + ) + with pytest.raises(RuntimeError, match="would run on the CPU"): + to_espnn_edge(program, allow_cpu_fallback=False) + + +def test_fallback_mode_reports_rejected_regions(monkeypatch): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + monkeypatch.setattr( + espnn_partitioner, "_boundary_is_valid", lambda nodes, resolver: False + ) + _, report = to_espnn_edge(program) + assert report["partitions"] == 0 + assert report["rejected_partitions"] >= 1 + assert report["cpu_ops"] + + +def _weight_observer(prepared, conv_name_fragment): + modules = dict(prepared.named_modules()) + for node in prepared.graph.nodes: + if node.op != "call_function" or "conv" not in str(node.target): + continue + stack = node.meta.get("nn_module_stack", {}) + if not any(conv_name_fragment in path for path, _ in stack.values()): + continue + weight = node.args[1] + observer = weight if weight.op == "call_module" else weight.args[0] + return type(modules[str(observer.target)]).__name__ + raise AssertionError(f"no conv under {conv_name_fragment}") + + +def test_module_name_config_reaches_fused_conv_relu(): + model = FeaturesAndClassifier().eval() + example = (torch.randn(1, 3, 8, 8),) + per_tensor = get_symmetric_quantization_config(is_per_channel=False) + quantizer = EspNNQuantizer().set_module_name("classifier", per_tensor) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), quantizer + ) + assert "PerChannel" in _weight_observer(prepared, "features") + assert "PerChannel" not in _weight_observer(prepared, "classifier") + + +def test_hardtanh_bounds_given_as_kwargs_fuse_as_relu6(): + graph = torch.fx.Graph() + x = graph.placeholder("x") + by_kwargs = graph.call_function( + torch.ops.aten.hardtanh.default, (x,), {"min_val": 0.0, "max_val": 6.0} + ) + by_args = graph.call_function(torch.ops.aten.hardtanh.default, (x, 0.0, 6.0)) + other = graph.call_function(torch.ops.aten.hardtanh.default, (x,)) + assert fused_activation_kind(by_kwargs) == "relu6" + assert fused_activation_kind(by_args) == "relu6" + assert fused_activation_kind(other) is None + + +def test_preprocess_rejects_unknown_format_spec(): + with pytest.raises(ValueError, match="espnn_format"): + EspNNBackend.preprocess(None, [CompileSpec(INT8_FORMAT_SPEC.key, b"float")]) + + +def test_runtime_limits_accept_a_lowered_graph(): + check_runtime_limits(_small_graph()) + + +def test_runtime_limits_reject_too_many_outputs(): + graph = _small_graph() + graph = dataclasses.replace(graph, outputs=list(range(MAX_GRAPH_IO + 1))) + with pytest.raises(ValueError, match="outputs"): + check_runtime_limits(graph) + + +def test_runtime_limits_reject_too_many_operations(): + graph = _small_graph() + operations = graph.operations * (MAX_GRAPH_OPERATIONS // len(graph.operations) + 1) + with pytest.raises(ValueError, match="operations"): + check_runtime_limits(dataclasses.replace(graph, operations=operations)) + + +def test_runtime_limits_reject_v6_opcodes_in_v5_graphs(): + graph = _small_graph() + copy = dataclasses.replace(graph.operations[0], opcode=XTS3OpCode.COPY) + with pytest.raises(ValueError, match="version 6"): + check_runtime_limits( + dataclasses.replace( + graph, version=5, operations=[copy], inputs=[], outputs=[] + ) + ) + + +def test_runtime_limits_reject_misaligned_int32_constants(): + graph = _small_graph() + tensors = [ + ( + dataclasses.replace(tensor, data_offset=tensor.data_offset + 2) + if tensor.flags & TENSOR_FLAG_CONSTANT and tensor.dtype == XTS3DType.INT32 + else tensor + ) + for tensor in graph.tensors + ] + with pytest.raises(ValueError, match="4-byte aligned"): + check_runtime_limits(dataclasses.replace(graph, tensors=tensors)) diff --git a/backends/espressif/test/test_cpu_fallback.py b/backends/espressif/test/test_cpu_fallback.py index c8db1f32f71..9c8746a1fb6 100644 --- a/backends/espressif/test/test_cpu_fallback.py +++ b/backends/espressif/test/test_cpu_fallback.py @@ -10,8 +10,7 @@ Host tests for partial delegation: int8 regions run on the ESP-NN delegate, unsupported operators stay on the ExecuTorch CPU (portable) kernels. - PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ - /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_fallback.py + pytest backends/espressif/test/test_cpu_fallback.py """ from __future__ import annotations diff --git a/backends/espressif/test/test_fold_batch_norm.py b/backends/espressif/test/test_fold_batch_norm.py index e1c07632700..dac2e796473 100644 --- a/backends/espressif/test/test_fold_batch_norm.py +++ b/backends/espressif/test/test_fold_batch_norm.py @@ -6,6 +6,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +import pytest import torch import torch.nn as nn @@ -13,7 +14,8 @@ count_conv_bn, fold_batch_norm, ) -from torchvision import models + +models = pytest.importorskip("torchvision.models") torch.set_grad_enabled(False) diff --git a/backends/espressif/test/test_int8_lowering.py b/backends/espressif/test/test_int8_lowering.py index c16ff1409b5..8522d711233 100644 --- a/backends/espressif/test/test_int8_lowering.py +++ b/backends/espressif/test/test_int8_lowering.py @@ -9,8 +9,7 @@ """ Host tests for the ESP32 int8 quantize-and-lower pipeline. - PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ - /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_int8.py + pytest backends/espressif/test/test_int8_lowering.py """ from __future__ import annotations @@ -18,8 +17,11 @@ import random import struct -import numpy as np import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + import torch import torch.nn as nn import torch.nn.functional as F diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py index 469282a7a39..9429e4d2ba5 100644 --- a/backends/espressif/test/test_int8_ops.py +++ b/backends/espressif/test/test_int8_ops.py @@ -11,14 +11,16 @@ transposed and dilated convolution, broadcasting, padding, any-axis concat, split, and multiple graph inputs and outputs. - PYTHONPATH=/home/mwaniki/Documents/torchenv/lib/python3.12/site-packages \ - /usr/bin/python3.12 -s -m pytest -q tools/tests/test_esp32_ops_v6.py + pytest backends/espressif/test/test_int8_ops.py """ from __future__ import annotations -import numpy as np import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + import torch import torch.nn as nn import torch.nn.functional as F @@ -221,7 +223,25 @@ def forward(self, x): torch.manual_seed(20261014) + + +class GroupedConvs(nn.Module): + """Grouped (not depthwise) convolutions: plain, strided and dilated.""" + + def __init__(self): + super().__init__() + self.plain = nn.Conv2d(16, 16, 3, padding=1, groups=4) + self.strided = nn.Conv2d(16, 24, 3, stride=2, padding=1, groups=8) + self.dilated = nn.Conv2d(24, 24, 3, padding=2, dilation=2, groups=3) + + def forward(self, x): + x = F.relu(self.plain(x)) + x = F.relu(self.strided(x)) + return self.dilated(x) + + CASES = [ + ("grouped-conv", GroupedConvs(), [(1, 16, 12, 12)], 2), ("detection-head-multi-output", DetectionHead(), [(1, 3, 16, 16)], 2), ("segmentation-dilated-transposed", SegmentationDecoder(), [(1, 3, 16, 16)], 3), ("upsample-nearest-x2", UpsampleModel("nearest"), [(1, 3, 16, 16)], 2), diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py new file mode 100644 index 00000000000..00077357b9a --- /dev/null +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -0,0 +1,170 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Builds the device-side XTS3 parser (espnn_runtime/src/xts3_graph.cpp, which +has no ESP-IDF dependencies) for the host and checks that it accepts graphs +produced by the serializer and rejects corrupted ones: overflowing COPY +extents and segment counts, oversized UPSAMPLE tables and misaligned int32 +constants. Skipped when no C++ compiler is available. + + pytest backends/espressif/test/test_xts3_parser_host.py +""" + +from __future__ import annotations + +import os +import shutil +import struct +import subprocess +from pathlib import Path + +import pytest + +pytest.importorskip("torchao") + +from executorch.backends.espressif.serialization import xts3 # noqa: E402 +from executorch.backends.espressif.test import test_int8_ops as ops # noqa: E402 + +RUNTIME = Path(__file__).resolve().parents[1] / "espnn_runtime" +DRIVER = Path(__file__).resolve().parent / "host" / "xts3_parse_main.cpp" +INT32_MAX = 2**31 - 1 + + +@pytest.fixture(scope="module") +def parser(tmp_path_factory): + compiler = os.environ.get("CXX") or shutil.which("c++") or shutil.which("g++") + if compiler is None: + pytest.skip("no C++ compiler") + binary = tmp_path_factory.mktemp("xts3") / "xts3_parse" + sources = [str(RUNTIME / "src" / "xts3_graph.cpp"), str(DRIVER)] + base = [compiler, "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror"] + base += ["-I", str(RUNTIME / "include"), *sources, "-o", str(binary)] + sanitized = base[:1] + ["-fsanitize=address,undefined"] + base[1:] + for command in (sanitized, base): + if subprocess.run(command, capture_output=True).returncode == 0: + break + else: + result = subprocess.run(base, capture_output=True, text=True) + pytest.fail(f"host build of xts3_graph.cpp failed:\n{result.stderr}") + + def run(blobs: dict[str, bytes]) -> dict[str, str]: + folder = binary.parent + paths = [] + for name, blob in blobs.items(): + path = folder / f"{name}.xts3" + path.write_bytes(blob) + paths.append(str(path)) + result = subprocess.run( + [str(binary), *paths], + capture_output=True, + text=True, + env={**os.environ, "UBSAN_OPTIONS": "halt_on_error=1"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + verdicts = {} + for line in result.stdout.splitlines(): + if line.startswith("VERDICT "): + path, verdict = line[len("VERDICT ") :].rsplit(" ", 1) + verdicts[Path(path).stem] = verdict + return verdicts + + return run + + +def _lowered(model, shapes) -> xts3.XTS3Graph: + _, program = ops.quantize(model, shapes) + return xts3.build_graph(program) + + +def _layout(graph: xts3.XTS3Graph): + io_table = 0 + if graph.version >= 6: + io_table = xts3.IO_TABLE_HEADER_SIZE + 4 * ( + len(graph.inputs) + len(graph.outputs) + ) + tensors = xts3.HEADER_SIZE + io_table + operations = tensors + xts3.TENSOR_SIZE * len(graph.tensors) + parameters = operations + xts3.OP_SIZE * len(graph.operations) + offsets, cursor = [], parameters + for operation in graph.operations: + offsets.append(cursor) + cursor += len(operation.params) + return tensors, offsets + + +def _first(graph, opcode) -> int: + return next(i for i, op in enumerate(graph.operations) if op.opcode == opcode) + + +def _patch(blob: bytes, offset: int, fmt: str, *values) -> bytes: + data = bytearray(blob) + struct.pack_into(fmt, data, offset, *values) + return bytes(data) + + +def test_parser_accepts_serialized_graphs(parser): + blobs = { + name: xts3.serialize_graph(_lowered(model, shapes)) + for name, model, shapes, _ in ops.CASES + } + assert set(parser(blobs).values()) == {"accept"} + + +def test_parser_rejects_copy_extent_overflow(parser): + graph = _lowered(ops.TransposeSelect(), [(1, 3, 4, 5)]) + blob = xts3.serialize_graph(graph) + _, offsets = _layout(graph) + segment = offsets[_first(graph, xts3.XTS3OpCode.COPY)] + xts3.COPY_HEADER_SIZE + big = [INT32_MAX] * 3 + [1] * 3 + corrupted = _patch(blob, segment + 4, " int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument("--port", default=os.environ.get("ESP_PORT", "/dev/ttyACM0")) + parser.add_argument( + "--port", + default=os.environ.get("ESPPORT"), + required="ESPPORT" not in os.environ, + help="serial port of the board (default: $ESPPORT)", + ) parser.add_argument( "--labels", type=Path, default=TOOLS_DIR / "imagenet_classes.txt" ) diff --git a/examples/espressif/espnn_runner/main/CMakeLists.txt b/examples/espressif/espnn_runner/main/CMakeLists.txt index 56c8efdc9c5..025ffd7b69f 100644 --- a/examples/espressif/espnn_runner/main/CMakeLists.txt +++ b/examples/espressif/espnn_runner/main/CMakeLists.txt @@ -8,7 +8,9 @@ set(runner_sources "main.cpp" "int8_suite.cpp" "bundle_data.S") if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) - list(APPEND runner_sources "pte_suite.cpp") + list(APPEND runner_sources "pte_suite.cpp" + "${CMAKE_CURRENT_LIST_DIR}/../../executor_runner/esp_pal.cpp" + ) endif() idf_component_register( diff --git a/examples/espressif/espnn_runner/sdkconfig.emulator b/examples/espressif/espnn_runner/sdkconfig.emulator new file mode 100644 index 00000000000..75f258b4983 --- /dev/null +++ b/examples/espressif/espnn_runner/sdkconfig.emulator @@ -0,0 +1 @@ +CONFIG_NN_ANSI_C=y diff --git a/examples/espressif/scripts/benchmark.py b/examples/espressif/scripts/benchmark.py index fc857aa1f61..d87e4a195b5 100644 --- a/examples/espressif/scripts/benchmark.py +++ b/examples/espressif/scripts/benchmark.py @@ -17,7 +17,7 @@ Run inside an activated ESP-IDF shell with a Python that has executorch: python examples/espressif/scripts/benchmark.py --port /dev/ttyACM0 \ - --images ~/data/imagenette2-160 --dashboard + --images path/to/imagenette2-160 --dashboard Models that need more RAM than the board has are reported as "did not fit". ExecuTorch-mode models need ExecuTorch cross-compiled for the target @@ -29,6 +29,7 @@ import argparse import json +import os import re import subprocess import sys @@ -339,7 +340,12 @@ def main() -> int: # noqa: C901 parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument("--port", default="/dev/ttyACM0") + parser.add_argument( + "--port", + default=os.environ.get("ESPPORT"), + required="ESPPORT" not in os.environ, + help="serial port of the board (default: $ESPPORT)", + ) parser.add_argument( "--target", help="override the detected chip (esp32s3, esp32, ...)" ) diff --git a/examples/espressif/scripts/espnn_board.py b/examples/espressif/scripts/espnn_board.py index 7982f655a0f..46975c87788 100644 --- a/examples/espressif/scripts/espnn_board.py +++ b/examples/espressif/scripts/espnn_board.py @@ -294,4 +294,7 @@ def run_bundle( if __name__ == "__main__": - print(detect_chip(sys.argv[1] if len(sys.argv) > 1 else "/dev/ttyACM0")) + port = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("ESPPORT") + if not port: + sys.exit("usage: espnn_board.py PORT (or set ESPPORT)") + print(detect_chip(port)) diff --git a/examples/espressif/scripts/make_test_bundles.py b/examples/espressif/scripts/make_test_bundles.py index ba05686a1b3..20987818986 100644 --- a/examples/espressif/scripts/make_test_bundles.py +++ b/examples/espressif/scripts/make_test_bundles.py @@ -151,12 +151,21 @@ def main() -> int: default=Path(__file__).resolve().parents[1] / "espnn_runner" / "bundles", ) parser.add_argument("--samples", type=int, default=3) + parser.add_argument( + "--suites", + default="op,pte", + help="comma separated: op (standalone op_suite.xtb), pte (pte_suite.xtp)", + ) args = parser.parse_args() + suites = set(args.suites.split(",")) torch.set_grad_enabled(False) - for filename, magic, builder in ( - ("op_suite.xtb", b"XTB1", build_op_suite), - ("pte_suite.xtp", b"XTP1", build_pte_suite), + args.output_dir.mkdir(parents=True, exist_ok=True) + for suite, filename, magic, builder in ( + ("op", "op_suite.xtb", b"XTB1", build_op_suite), + ("pte", "pte_suite.xtp", b"XTP1", build_pte_suite), ): + if suite not in suites: + continue entries, reports = builder(args.samples) path = args.output_dir / filename size = write_bundle(path, magic, entries) diff --git a/examples/espressif/scripts/run_bundle.py b/examples/espressif/scripts/run_bundle.py index 0eb9ae0c878..367d1233ed7 100644 --- a/examples/espressif/scripts/run_bundle.py +++ b/examples/espressif/scripts/run_bundle.py @@ -18,6 +18,7 @@ import argparse import json +import os import sys from pathlib import Path @@ -30,7 +31,12 @@ def main() -> int: description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("bundle", type=Path) - parser.add_argument("--port", default="/dev/ttyACM0") + parser.add_argument( + "--port", + default=os.environ.get("ESPPORT"), + required="ESPPORT" not in os.environ, + help="serial port of the board (default: $ESPPORT)", + ) parser.add_argument("--target", help="override the detected chip") parser.add_argument("--iterations", type=int, default=10) parser.add_argument("--timeout", type=float, default=600) diff --git a/pytest.ini b/pytest.ini index 5ef01fde9a0..04b40dc5df9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -85,6 +85,7 @@ testpaths = # backends backends/apple/coreml/test + backends/espressif/test backends/native/test backends/test/harness/tests backends/test/suite/tests From 450b7517376cb94c890a8d406611921debc28f78 Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Mon, 19 Oct 2026 09:10:56 +0200 Subject: [PATCH 05/10] Use existing passes instead of reimplemented ones Drop _passes/fold_batch_norm and _passes/remove_dropout: PT2E already folds conv+BN, and dropout is a no-op in exported eval models. Replace DuplicateQuantizationFanoutPass with the existing DuplicateDequantNodePass. --- backends/espressif/README.md | 5 +- backends/espressif/_passes/__init__.py | 12 --- backends/espressif/_passes/fold_batch_norm.py | 79 ----------------- backends/espressif/_passes/remove_dropout.py | 45 ---------- backends/espressif/lowering.py | 10 +-- backends/espressif/partition/__init__.py | 4 +- .../espressif/partition/espnn_partitioner.py | 41 --------- .../espressif/test/test_fold_batch_norm.py | 87 ------------------- examples/espressif/aot_espnn_compiler.py | 5 +- 9 files changed, 11 insertions(+), 277 deletions(-) delete mode 100644 backends/espressif/_passes/__init__.py delete mode 100644 backends/espressif/_passes/fold_batch_norm.py delete mode 100644 backends/espressif/_passes/remove_dropout.py delete mode 100644 backends/espressif/test/test_fold_batch_norm.py diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 543df9f4ba5..6ce89955349 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -27,7 +27,6 @@ backends/espressif/ lowering.py to_espnn_edge(): export -> edge -> lowered program + report serialization/ XTS3 format writer and parser, op lowering rules reference/ bit-exact host model of the ESP-NN kernels and the CPU fallback - _passes/ batch-norm folding, dropout removal espnn_runtime/ ESP-IDF component: XTS3 interpreter + ExecuTorch BackendInterface test/ pytest suite (lowering, operators, CPU fallback) examples/espressif/ @@ -53,7 +52,9 @@ symmetric weights, as annotated by `EspNNQuantizer`. | Resize | `upsample_nearest2d`, `upsample_bilinear2d` (`align_corners` either way) | | Data movement | `reshape`, `view`, `flatten`, `unflatten`, `permute`, `transpose`, `squeeze`, `unsqueeze`, `slice`, `select`, `expand`, `pad`, `cat`, `stack`, `split`, `chunk`, `unbind` | -Batch norm is folded into the preceding convolution before quantization. +Batch norm that follows a convolution is folded into it by PT2E +(`prepare_pt2e` / `prepare_qat_pt2e`); dropout in an exported eval-mode +model is a no-op. Partitions can have several inputs and outputs. Anything else stays in the program and runs on the CPU, including float ops, LSTM, layer norm and attention matmuls. The partitioner reports what it delegated and what it left diff --git a/backends/espressif/_passes/__init__.py b/backends/espressif/_passes/__init__.py deleted file mode 100644 index beda70b1315..00000000000 --- a/backends/espressif/_passes/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -from .fold_batch_norm import fold_batch_norm -from .remove_dropout import remove_dropout - -__all__ = ["fold_batch_norm", "remove_dropout"] diff --git a/backends/espressif/_passes/fold_batch_norm.py b/backends/espressif/_passes/fold_batch_norm.py deleted file mode 100644 index f3251ae2e87..00000000000 --- a/backends/espressif/_passes/fold_batch_norm.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -from collections import OrderedDict -from copy import deepcopy - -import torch.nn as nn -from torch.nn.utils.fusion import fuse_conv_bn_eval - - -def _fuse_sequential(module: nn.Sequential) -> tuple[nn.Sequential, int]: - """Fold adjacent Conv2d -> BatchNorm2d pairs inside one nn.Sequential.""" - children = list(module.named_children()) - fused = OrderedDict() - count = 0 - index = 0 - while index < len(children): - name, child = children[index] - if ( - isinstance(child, nn.Conv2d) - and index + 1 < len(children) - and isinstance(children[index + 1][1], nn.BatchNorm2d) - ): - bn_name, batch_norm = children[index + 1] - if child.training or batch_norm.training: - raise RuntimeError( - f"{name} / {bn_name} must be in eval() mode to fold batch norm" - ) - fused[name] = fuse_conv_bn_eval(child, batch_norm) - count += 1 - index += 2 - continue - fused[name] = child - index += 1 - return nn.Sequential(fused), count - - -def _fuse_recursive(module: nn.Module) -> int: - total = 0 - for name, child in list(module.named_children()): - if isinstance(child, nn.Sequential): - fused_child, count = _fuse_sequential(child) - setattr(module, name, fused_child) - total += count + _fuse_recursive(fused_child) - else: - total += _fuse_recursive(child) - return total - - -def fold_batch_norm( - model: nn.Module, *, inplace: bool = False -) -> tuple[nn.Module, int]: - """ - Fold inference-time Conv2d -> BatchNorm2d pairs held in nn.Sequential - containers (the torchvision convention) before export, so that the - quantizer sees plain convolutions. Returns (model, number_of_folds). - """ - if model.training: - raise ValueError("fold_batch_norm() requires model.eval()") - if not inplace: - model = deepcopy(model) - return model, _fuse_recursive(model) - - -def count_conv_bn(model: nn.Module) -> tuple[int, int]: - """Return (number of Conv2d modules, number of BatchNorm2d modules).""" - convs = sum(isinstance(m, nn.Conv2d) for m in model.modules()) - norms = sum(isinstance(m, nn.BatchNorm2d) for m in model.modules()) - return convs, norms - - -__all__ = ["count_conv_bn", "fold_batch_norm"] diff --git a/backends/espressif/_passes/remove_dropout.py b/backends/espressif/_passes/remove_dropout.py deleted file mode 100644 index 60c0e40a3d3..00000000000 --- a/backends/espressif/_passes/remove_dropout.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -from __future__ import annotations - -import torch.nn as nn - - -def remove_dropout(model: nn.Module) -> nn.Module: - """ - Replace every nn.Dropout module with nn.Identity(). - - The top-level model must be in eval() mode. - """ - - if model.training: - raise ValueError("remove_dropout() requires model.eval()") - - for name, child in list(model.named_children()): - if isinstance(child, nn.Dropout): - setattr(model, name, nn.Identity()) - else: - remove_dropout_from_children(child) - - return model - - -def remove_dropout_from_children(module: nn.Module) -> None: - """ - Recursively replace Dropout modules. - - Deliberately does NOT check module.training. - The top-level model has already been checked. - """ - - for name, child in list(module.named_children()): - if isinstance(child, nn.Dropout): - setattr(module, name, nn.Identity()) - else: - remove_dropout_from_children(child) diff --git a/backends/espressif/lowering.py b/backends/espressif/lowering.py index cad9b29514b..67e4063b54d 100644 --- a/backends/espressif/lowering.py +++ b/backends/espressif/lowering.py @@ -10,11 +10,11 @@ import executorch.kernels.quantized # noqa: F401 registers quantize/dequantize out variants -from executorch.backends.espressif.partition.espnn_partitioner import ( - DuplicateQuantizationFanoutPass, - EspNNPartitioner, -) +from executorch.backends.espressif.partition.espnn_partitioner import EspNNPartitioner from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower +from executorch.exir.backend.canonical_partitioners.duplicate_dequant_node_pass import ( + DuplicateDequantNodePass, +) def to_espnn_edge(program, allow_cpu_fallback: bool = True): @@ -26,7 +26,7 @@ def to_espnn_edge(program, allow_cpu_fallback: bool = True): partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback) edge = to_edge_transform_and_lower( program, - transform_passes=[DuplicateQuantizationFanoutPass()], + transform_passes=[DuplicateDequantNodePass()], partitioner=[partitioner], compile_config=EdgeCompileConfig(_check_ir_validity=False), ) diff --git a/backends/espressif/partition/__init__.py b/backends/espressif/partition/__init__.py index 5af6d98f28c..3b10d5189d8 100644 --- a/backends/espressif/partition/__init__.py +++ b/backends/espressif/partition/__init__.py @@ -6,6 +6,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -from .espnn_partitioner import DuplicateQuantizationFanoutPass, EspNNPartitioner +from .espnn_partitioner import EspNNPartitioner -__all__ = ["DuplicateQuantizationFanoutPass", "EspNNPartitioner"] +__all__ = ["EspNNPartitioner"] diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py index 6b4bf9a6cd1..c1aa0de730e 100644 --- a/backends/espressif/partition/espnn_partitioner.py +++ b/backends/espressif/partition/espnn_partitioner.py @@ -42,7 +42,6 @@ from executorch.exir.backend.utils import tag_constant_data from torch.export import ExportedProgram from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner -from torch.fx.passes.infra.pass_base import PassBase, PassResult from torch.fx.passes.operator_support import OperatorSupportBase from torch.fx.passes.utils.fuser_utils import validate_partition @@ -140,44 +139,6 @@ def _is_passthrough(node) -> bool: ) -def duplicate_quantization_fanout(graph_module: torch.fx.GraphModule) -> int: - """ - Give every consumer its own quantize -> dequantize chain so that a - partition boundary can fall between any quantize and dequantize pair. - """ - graph = graph_module.graph - copies = 0 - for node in list(graph.nodes): - if not is_dequantize(node) or len(node.users) < 2: - continue - for user in list(node.users)[1:]: - with graph.inserting_after(node): - duplicate = graph.node_copy(node) - duplicate.meta = dict(node.meta) - user.replace_input_with(node, duplicate) - copies += 1 - for node in list(graph.nodes): - if not is_quantize(node) or len(node.users) < 2: - continue - for user in list(node.users)[1:]: - with graph.inserting_after(node): - duplicate = graph.node_copy(node) - duplicate.meta = dict(node.meta) - user.replace_input_with(node, duplicate) - copies += 1 - if copies: - graph.lint() - graph_module.recompile() - return copies - - -class DuplicateQuantizationFanoutPass(PassBase): - """Run before partitioning (to_edge_transform_and_lower transform_passes).""" - - def call(self, graph_module: torch.fx.GraphModule) -> PassResult: - return PassResult(graph_module, duplicate_quantization_fanout(graph_module) > 0) - - class StructuralResolver(ExportedTensorResolver): """ ExecuTorch partitions a copy of the program whose constants are @@ -432,10 +393,8 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult: __all__ = [ - "DuplicateQuantizationFanoutPass", "StructuralResolver", "EspNNPartitioner", - "duplicate_quantization_fanout", "find_supported_nodes", "is_supported_int8_node", ] diff --git a/backends/espressif/test/test_fold_batch_norm.py b/backends/espressif/test/test_fold_batch_norm.py deleted file mode 100644 index dac2e796473..00000000000 --- a/backends/espressif/test/test_fold_batch_norm.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -import pytest -import torch -import torch.nn as nn - -from executorch.backends.espressif._passes.fold_batch_norm import ( - count_conv_bn, - fold_batch_norm, -) - -models = pytest.importorskip("torchvision.models") - - -torch.set_grad_enabled(False) - - -class ConvBnStack(nn.Module): - def __init__(self): - super().__init__() - self.features = nn.Sequential( - nn.Conv2d(3, 8, 3, padding=1, bias=False), - nn.BatchNorm2d(8), - nn.ReLU6(), - nn.Conv2d(8, 8, 3, padding=1, groups=8), - nn.BatchNorm2d(8), - nn.ReLU(), - ) - - def forward(self, x): - return self.features(x) - - -def _randomize_batch_norm(model: nn.Module) -> None: - generator = torch.Generator().manual_seed(0) - for module in model.modules(): - if isinstance(module, nn.BatchNorm2d): - module.running_mean.copy_( - torch.randn(module.num_features, generator=generator) * 0.1 - ) - module.running_var.copy_( - torch.rand(module.num_features, generator=generator) + 0.5 - ) - module.weight.copy_( - torch.rand(module.num_features, generator=generator) + 0.5 - ) - module.bias.copy_( - torch.randn(module.num_features, generator=generator) * 0.1 - ) - - -def test_folding_removes_batch_norm_and_preserves_outputs(): - model = ConvBnStack().eval() - _randomize_batch_norm(model) - x = torch.randn(2, 3, 16, 16) - expected = model(x) - folded, count = fold_batch_norm(model, inplace=False) - assert count == 2 - assert count_conv_bn(folded) == (2, 0) - assert count_conv_bn(model) == (2, 2) - assert torch.allclose(folded(x), expected, atol=1e-5) - - -def test_folding_mobilenet_v2_matches_eager(): - model = models.mobilenet_v2(weights=None).eval() - _randomize_batch_norm(model) - convs, norms = count_conv_bn(model) - folded, count = fold_batch_norm(model, inplace=False) - assert count == norms > 0 - assert count_conv_bn(folded) == (convs, 0) - x = torch.randn(1, 3, 96, 96) - assert torch.allclose(folded(x), model(x), atol=1e-4, rtol=1e-4) - - -def test_training_mode_is_rejected(): - model = ConvBnStack().train() - try: - fold_batch_norm(model) - except ValueError: - return - raise AssertionError("fold_batch_norm accepted a model in training mode") diff --git a/examples/espressif/aot_espnn_compiler.py b/examples/espressif/aot_espnn_compiler.py index ccc7137387b..ccd7aac4112 100644 --- a/examples/espressif/aot_espnn_compiler.py +++ b/examples/espressif/aot_espnn_compiler.py @@ -44,7 +44,6 @@ import numpy as np import torch -from executorch.backends.espressif._passes import fold_batch_norm, remove_dropout from executorch.backends.espressif.quantizer import EspNNQuantizer from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e @@ -472,9 +471,7 @@ def main() -> int: for s in (args.input_shape or ["1,3,224,224"]) ] print(f"[1/5] Loading {args.model}") - model = remove_dropout( - fold_batch_norm(load_model(args.model, args.weights).eval())[0] - ).eval() + model = load_model(args.model, args.weights).eval() calibration, evaluation = make_data(args, shapes) quantized, program = quantize(model, calibration, args) compile_fn = compile_standalone if args.mode == "standalone" else compile_executorch From 51b2bb569b7e6fea1d7d15428677fa1192ab10a8 Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Tue, 27 Oct 2026 14:26:42 +0100 Subject: [PATCH 06/10] Run ESP-NN e2e tests through the ExecuTorch runtime in the emulator Export the backend's test models as BundleIO programs and run each one through examples/espressif/project, built with -DET_ESPNN_BACKEND=ON, in esp-emu. This replaces the standalone XTS3 operator suite in CI. The espnn_runtime component links the ExecuTorch targets that executor_runner imports, executor_runner links quantized_ops_lib, and the standalone runner, board scripts, benchmarks and compiler are removed. --- .ci/scripts/test_espressif.sh | 74 ++- backends/espressif/README.md | 292 +++------- .../espressif/espnn_runtime/CMakeLists.txt | 91 +-- backends/espressif/espnn_runtime/Kconfig | 12 +- examples/espressif/.gitignore | 2 - examples/espressif/aot_espnn_compiler.py | 515 ----------------- examples/espressif/benchmarks/esp32.json | 186 ------- examples/espressif/benchmarks/esp32.md | 17 - examples/espressif/benchmarks/esp32s3.json | 370 ------------ examples/espressif/benchmarks/esp32s3.md | 20 - examples/espressif/espnn_bundle.py | 153 ----- .../espressif/espnn_runner/CMakeLists.txt | 17 - examples/espressif/espnn_runner/README.md | 49 -- .../espnn_runner/main/CMakeLists.txt | 47 -- .../espnn_runner/main/Kconfig.projbuild | 51 -- .../espressif/espnn_runner/main/bundle_data.S | 20 - .../espnn_runner/main/int8_suite.cpp | 324 ----------- examples/espressif/espnn_runner/main/main.cpp | 89 --- .../espressif/espnn_runner/main/pte_suite.cpp | 494 ---------------- examples/espressif/espnn_runner/main/runner.h | 48 -- .../espnn_runner/partitions_16mb.csv | 3 - .../espressif/espnn_runner/partitions_4mb.csv | 3 - .../espressif/espnn_runner/sdkconfig.defaults | 11 - .../espnn_runner/sdkconfig.defaults.esp32 | 5 - .../espnn_runner/sdkconfig.defaults.esp32s3 | 9 - .../espressif/espnn_runner/sdkconfig.emulator | 1 - .../espnn_runner/sdkconfig.executorch | 1 - .../espressif/executor_runner/CMakeLists.txt | 5 + .../espressif/export_espnn_test_programs.py | 155 ++++++ examples/espressif/project/CMakeLists.txt | 8 + examples/espressif/scripts/bench_models.py | 195 ------- examples/espressif/scripts/benchmark.py | 526 ------------------ examples/espressif/scripts/board_dashboard.py | 308 ---------- .../espressif/scripts/build_executorch_esp.sh | 80 --- examples/espressif/scripts/espnn_board.py | 300 ---------- examples/espressif/scripts/license_headers.py | 140 ----- .../espressif/scripts/make_test_bundles.py | 178 ------ examples/espressif/scripts/run_bundle.py | 85 --- examples/espressif/test_emulator.py | 4 +- 39 files changed, 328 insertions(+), 4560 deletions(-) delete mode 100644 examples/espressif/aot_espnn_compiler.py delete mode 100644 examples/espressif/benchmarks/esp32.json delete mode 100644 examples/espressif/benchmarks/esp32.md delete mode 100644 examples/espressif/benchmarks/esp32s3.json delete mode 100644 examples/espressif/benchmarks/esp32s3.md delete mode 100644 examples/espressif/espnn_bundle.py delete mode 100644 examples/espressif/espnn_runner/CMakeLists.txt delete mode 100644 examples/espressif/espnn_runner/README.md delete mode 100644 examples/espressif/espnn_runner/main/CMakeLists.txt delete mode 100644 examples/espressif/espnn_runner/main/Kconfig.projbuild delete mode 100644 examples/espressif/espnn_runner/main/bundle_data.S delete mode 100644 examples/espressif/espnn_runner/main/int8_suite.cpp delete mode 100644 examples/espressif/espnn_runner/main/main.cpp delete mode 100644 examples/espressif/espnn_runner/main/pte_suite.cpp delete mode 100644 examples/espressif/espnn_runner/main/runner.h delete mode 100644 examples/espressif/espnn_runner/partitions_16mb.csv delete mode 100644 examples/espressif/espnn_runner/partitions_4mb.csv delete mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults delete mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults.esp32 delete mode 100644 examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 delete mode 100644 examples/espressif/espnn_runner/sdkconfig.emulator delete mode 100644 examples/espressif/espnn_runner/sdkconfig.executorch create mode 100644 examples/espressif/export_espnn_test_programs.py delete mode 100644 examples/espressif/scripts/bench_models.py delete mode 100644 examples/espressif/scripts/benchmark.py delete mode 100644 examples/espressif/scripts/board_dashboard.py delete mode 100755 examples/espressif/scripts/build_executorch_esp.sh delete mode 100644 examples/espressif/scripts/espnn_board.py delete mode 100644 examples/espressif/scripts/license_headers.py delete mode 100644 examples/espressif/scripts/make_test_bundles.py delete mode 100644 examples/espressif/scripts/run_bundle.py diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index ac630a2a967..50a65503757 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -34,13 +34,24 @@ source "${ESPRESSIF_TOOLS_DIR}/setup_path.sh" python examples/espressif/export_smoke_model.py \ --output "${build_dir}/smoke.bpte" 2>&1 | tee "${build_dir}/export.log" +python examples/espressif/export_espnn_test_programs.py \ + --output-dir "${build_dir}/espnn-tests" 2>&1 | tee "${build_dir}/espnn-export.log" + +# One runtime serves the smoke test and the ESP-NN programs. quantized_ops_lib +# registers kernels outside the selected list, so the registry is sized +# explicitly instead of from the list. The quantized kernels' parallel_for +# runs inline without pthreadpool, which needs POSIX calls ESP-IDF lacks. +espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \ + | tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -) cmake --preset esp-baremetal -G Ninja -B "${build_dir}/runtime" \ -DCMAKE_TOOLCHAIN_FILE="${IDF_PATH}/tools/cmake/toolchain-esp32s3.cmake" \ -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_DEVTOOLS=ON \ - -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=OFF \ - -DEXECUTORCH_SELECT_OPS_LIST=aten::add.out,aten::mul.out \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_SELECT_OPS_LIST="aten::add.out,aten::mul.out,${espnn_ops}" \ + -DMAX_KERNEL_NUM=256 \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF -DEXECUTORCH_BUILD_CPUINFO=OFF \ 2>&1 | tee "${build_dir}/runtime-configure.log" cmake --build "${build_dir}/runtime" \ 2>&1 | tee "${build_dir}/runtime-build.log" @@ -67,21 +78,44 @@ timeout 130s esp-emu --chip esp32s3 \ 2>&1 | tee "${build_dir}/emulator.log" python examples/espressif/test_emulator.py "${build_dir}/emulator.log" -# ESP-NN backend: run the standalone operator suite and compare every case -# bit-exactly with the host reference. The emulator does not implement the -# ESP32-S3 SIMD instructions, so ESP-NN is built with its ANSI C kernels. -python examples/espressif/scripts/make_test_bundles.py --suites op \ - --output-dir "${build_dir}/espnn-bundles" 2>&1 | tee "${build_dir}/espnn-bundles.log" -printf 'CONFIG_ESPNN_RUNNER_BUNDLE="%s"\n' "${build_dir}/espnn-bundles/op_suite.xtb" \ - > "${build_dir}/espnn-bundle.defaults" -idf.py -C examples/espressif/espnn_runner -B "${build_dir}/espnn" \ - -DIDF_TARGET=esp32s3 \ - -DSDKCONFIG="${build_dir}/espnn-sdkconfig" \ - "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;sdkconfig.emulator;${build_dir}/espnn-bundle.defaults" \ - build merge-bin 2>&1 | tee "${build_dir}/espnn-build.log" -timeout 330s esp-emu --chip esp32s3 \ - --firmware "${build_dir}/espnn/merged-binary.bin" \ - --elf "${build_dir}/espnn/espnn_runner.elf" \ - --net user --psram-size 8M --timeout 300s --exit-on 'ESPNN RUNNER:' \ - 2>&1 | tee "${build_dir}/espnn-emulator.log" -grep -q "ESPNN RUNNER: PASS" "${build_dir}/espnn-emulator.log" +# ESP-NN backend: run every exported program through the ExecuTorch runtime +# and the ESP-NN delegate, and check it against the bundled host reference. +# The emulator does not implement the ESP32-S3 SIMD instructions, so ESP-NN is +# built with its ANSI C kernels. +printf 'CONFIG_NN_ANSI_C=y\n' > "${build_dir}/espnn-emulator.defaults" +# Runs a command with its output in a log, and shows the log's tail on failure. +logged() { + local log="${build_dir}/$1" + shift + if ! "$@" > "${log}" 2>&1; then + tail -n 60 "${log}" + echo "FAIL: see ${log}" >&2 + return 1 + fi +} +while read -r -u 3 name atol rtol; do + logged "espnn-${name}-configure.log" \ + idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" \ + -DIDF_TARGET=esp32s3 \ + -DSDKCONFIG="${build_dir}/espnn-sdkconfig" \ + "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/espnn-emulator.defaults" \ + -DET_ESPNN_BACKEND=ON \ + -DET_BUILD_DIR_PATH="${build_dir}/runtime" \ + -DET_PTE_FILE_PATH="${build_dir}/espnn-tests/${name}.bpte" \ + -DET_BUNDLE_IO=ON -DET_NUM_INFERENCES=1 \ + -DET_ATOL="${atol}" -DET_RTOL="${rtol}" reconfigure + logged "espnn-${name}-build.log" cmake --build "${build_dir}/espnn-firmware" + logged "espnn-${name}-merge.log" \ + idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" merge-bin + logged "espnn-${name}-emulator.log" \ + timeout 90s esp-emu --chip esp32s3 \ + --firmware "${build_dir}/espnn-firmware/merged-binary.bin" \ + --elf "${build_dir}/espnn-firmware/executorch_esp_runner.elf" \ + --net user --psram-size 8M --timeout 60s --exit-on 'Program complete.' + echo -n "${name}: " + python examples/espressif/test_emulator.py "${build_dir}/espnn-${name}-emulator.log" +done 3< <(python -c ' +import json, sys +for entry in json.load(open(sys.argv[1])): + print(entry["name"], entry["atol"], entry["rtol"]) +' "${build_dir}/espnn-tests/manifest.json") diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 6ce89955349..3530f35345e 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -6,16 +6,12 @@ delegates quantized subgraphs to [ESP-NN](https://github.com/espressif/esp-nn), Espressif's kernel library. ESP-NN has SIMD kernels for the ESP32-S3 and portable C kernels for the other chips. -It has two ways to run a model: - -| Path | What runs on the chip | When to use it | -|---|---|---| -| **ExecuTorch delegate** | A normal `.pte` program. `EspNNBackend` runs the int8 partitions with ESP-NN, and ExecuTorch's portable kernels run everything else (float ops, unsupported layers, pre- and post-processing). | Any model that `torch.export` can capture, including models that only partly map to ESP-NN. | -| **Standalone int8 graph** | The same delegate payload (an XTS3 graph), run directly by the small `espnn_runtime` interpreter with no ExecuTorch runtime linked. | Models that are fully int8 end to end, when firmware size matters (the runner is about 330 KB). | - -The two paths share one lowering flow: the PT2E quantizer, the partitioner and -the XTS3 serializer. The host reference interpreter reproduces the device -arithmetic, so every test and example checks the board's output bit-exactly. +A lowered model is a normal `.pte` program. `EspNNBackend` runs the int8 +partitions with ESP-NN, and ExecuTorch's portable and quantized kernels run +everything else (float ops, unsupported layers, pre- and post-processing). +Each delegate payload is a compact int8 graph (XTS3) that the delegate's +interpreter executes. A host reference interpreter reproduces the device +arithmetic, so fully delegated programs are checked bit-exactly. ## Contents @@ -28,16 +24,17 @@ backends/espressif/ serialization/ XTS3 format writer and parser, op lowering rules reference/ bit-exact host model of the ESP-NN kernels and the CPU fallback espnn_runtime/ ESP-IDF component: XTS3 interpreter + ExecuTorch BackendInterface - test/ pytest suite (lowering, operators, CPU fallback) + test/ pytest suite (lowering, operators, CPU fallback, parser) examples/espressif/ - quantize_and_lower.py minimal quantize -> lower -> .pte example - aot_espnn_compiler.py full compiler CLI: calibration, QAT, accuracy, device test vectors - espnn_runner/ ESP-IDF app that runs and verifies bundles on a board - camera_stream/ live camera classification demo (ESP32-S3) - scripts/ cross-build, board runner, benchmarks, license headers - benchmarks/ benchmark results per chip + quantize_and_lower.py minimal quantize -> lower -> .pte example + export_espnn_test_programs.py the test models as BundleIO programs for the runner + camera_stream/ live camera classification on the ExecuTorch runtime ``` +Programs run on the board with the existing +[`examples/espressif/project`](../../examples/espressif/project) runner, built +with `-DET_ESPNN_BACKEND=ON`. + ## Supported operators All delegated operators are int8 with per-tensor activations and per-channel @@ -84,7 +81,7 @@ with torch.no_grad(): prepared(images) quantized = convert_pt2e(prepared) -# 3. Lower: int8 regions -> EspNNBackend delegates, the rest -> portable CPU kernels +# 3. Lower: int8 regions -> EspNNBackend delegates, the rest -> CPU kernels edge, report = to_espnn_edge(torch.export.export(quantized, example_inputs)) print(report["partitions"], report["delegated_ops"], report["cpu_ops"]) @@ -99,85 +96,73 @@ $ python examples/espressif/quantize_and_lower.py wrote mobilenet_v2_espnn.pte ``` -### The compiler CLI - -`aot_espnn_compiler.py` builds on the same steps for production use. It -calibrates on an image folder, can run quantization-aware training, and -reports float and int8 accuracy. It also writes a bundle that contains test -vectors, so the board can check its own output: - -```bash -# ExecuTorch program (.pte inside an .xtp bundle) with one epoch of QAT -python examples/espressif/aot_espnn_compiler.py \ - --model mobilenet_v3_small --mode executorch \ - --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val \ - --calib-samples 512 --qat-epochs 1 \ - --output mnv3s.xtp --pte-output mnv3s.pte - -# Standalone int8 graph (.xtb bundle), no ExecuTorch runtime on the device -python examples/espressif/aot_espnn_compiler.py \ - --model mobilenet_v3_small --mode standalone \ - --calib-dir imagenette2-160/train --output mnv3s.xtb - -# Your own model: path/to/file.py:factory, one --input-shape per input -python examples/espressif/aot_espnn_compiler.py \ - --model my_models.py:build_kws --weights none --input-shape 1,1,49,10 \ - --calib-npy kws_calib.npy --output kws.xtp -``` - -Tips: - -- Calibrate on a few hundred real samples. Random data gives useless scales. -- Use per-channel PTQ for MobileNetV2-style networks. Networks with hardswish - and squeeze-excite blocks (MobileNetV3, EfficientNet) lose more accuracy - with PTQ alone, and `--qat-epochs` recovers most of it. -- `--crc-outputs` makes the board compare a CRC of large outputs (such as - segmentation maps) instead of storing them in the bundle. +Calibrate on a few hundred real samples; random data gives useless scales. ## Running on the ESP32-S3 -You need ESP-IDF v5.1 or newer (tested with v5.5) and an ESP32-S3 board. The -examples assume 16 MB flash and 8 MB octal PSRAM, such as the -ESP32-S3-DevKitC-1 N16R8. Every command below runs in an activated ESP-IDF -shell (`. $IDF_PATH/export.sh`). +You need ESP-IDF (CI uses v6.1) and an activated ESP-IDF shell. +[`examples/espressif/setup.sh`](../../examples/espressif/setup.sh) installs a +matching ESP-IDF and the emulator. ### 1. Cross-compile ExecuTorch -Skip this step for the standalone path. +Build the runtime with the `esp-baremetal` preset. Select the portable CPU +operators your program needs. The quantized kernels provide the +quantize/dequantize operators at the delegate boundaries. The program lists +its operators: -```bash -examples/espressif/scripts/build_executorch_esp.sh esp32s3 -# -> cmake-out-esp-esp32s3/{lib,include} +```python +plan = edge.to_executorch().executorch_program.execution_plan[0] +print(",".join(sorted( + f"{op.name}.{op.overload}" for op in plan.operators + if not op.name.startswith("quantized_decomposed::") +))) ``` -The script uses the IDF toolchain file. It builds the ExecuTorch runtime, the -portable kernels and the quantized kernels as static libraries, with exceptions -and RTTI off and with `-mlongcalls`. Set `ET_SELECT_OPS_MODEL=model.pte` to -build only the kernels that a given program needs. +```bash +cmake --preset esp-baremetal -G Ninja -B cmake-out-esp \ + -DCMAKE_TOOLCHAIN_FILE="${IDF_PATH}/tools/cmake/toolchain-esp32s3.cmake" \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_DEVTOOLS=ON \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_SELECT_OPS_LIST= \ + -DMAX_KERNEL_NUM=256 \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF -DEXECUTORCH_BUILD_CPUINFO=OFF +cmake --build cmake-out-esp --target install +``` -### 2. Add the component to your application +- `quantized_ops_lib` registers kernels beyond the selected list, so + `MAX_KERNEL_NUM` is set explicitly instead of being sized from the + selection. +- pthreadpool needs POSIX calls that ESP-IDF doesn't provide. With it off, + the quantized kernels run single-threaded. +- The runner links `executorch_selected_kernels`, which only exists when a + selection is given. A fully delegated program with no CPU operators + still needs at least one selected operator, such as `aten::add.out`. -`backends/espressif/espnn_runtime` is a regular ESP-IDF component, and it -pulls in `espressif/esp-nn` from the component registry. Add it to your -project with `EXTRA_COMPONENT_DIRS`: +### 2. Build the runner with the ESP-NN backend -```cmake -set(EXTRA_COMPONENT_DIRS "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime") +```bash +idf.py -C examples/espressif/project -B build-espnn \ + -DIDF_TARGET=esp32s3 \ + -DET_ESPNN_BACKEND=ON \ + -DET_BUILD_DIR_PATH=$PWD/cmake-out-esp \ + -DET_PTE_FILE_PATH=$PWD/mobilenet_v2_espnn.pte \ + build flash monitor ``` -Then enable the delegate with `CONFIG_ESPNN_EXECUTORCH_DELEGATE=y`, using -menuconfig: *ExecuTorch ESP-NN backend*. The component links -`libexecutorch.a`, the kernel registries and the delegate from -`ESPNN_EXECUTORCH_BUILD_DIR`, which defaults to -`cmake-out-esp-${IDF_TARGET}`. It also registers `EspNNBackend` with the -runtime at startup. +`ET_ESPNN_BACKEND` adds +[`backends/espressif/espnn_runtime`](espnn_runtime) to the project. The +component pulls in `espressif/esp-nn` from the component registry, links +against the ExecuTorch targets that `executor_runner` imports, and registers +`EspNNBackend` at startup. It defines no platform layer; the runner's +`esp_pal.cpp` provides it. -The component does not define the ExecuTorch platform layer (`et_pal_*`). -Applications either compile the shared -[`examples/espressif/executor_runner/esp_pal.cpp`](../../examples/espressif/executor_runner/esp_pal.cpp), -as `espnn_runner` does, or provide their own. +For a BundleIO program (`.bpte`), add `-DET_BUNDLE_IO=ON` and the tolerances +`-DET_ATOL=... -DET_RTOL=...`. The runner then checks the outputs against the +bundled reference and logs `Test_result: PASS` or `FAIL`. -Other options: +Other options, in menuconfig under *ExecuTorch ESP-NN backend*: | Kconfig | Default | Effect | |---|---|---| @@ -185,129 +170,12 @@ Other options: | `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | | `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | -Load and run a program with the usual ExecuTorch API. `pte_data` points to -the program, for example embedded in flash with `EMBED_FILES` or read from a -partition: - -```cpp -#include -#include -#include - -using namespace executorch::runtime; - -runtime_init(); -executorch::extension::BufferDataLoader loader(pte_data, pte_size); -auto program = Program::load(&loader).get(); -auto meta = program.method_meta("forward").get(); -/* MemoryManager with the method's planned buffers (PSRAM on the S3), then: */ -auto method = program.load_method("forward", &memory_manager).get(); -method.set_input(input_tensor, 0); -method.execute(); -auto stats = executorch::backends::espressif::delegate_stats(); /* time spent inside ESP-NN delegates */ -``` - -[`examples/espressif/espnn_runner/main/pte_suite.cpp`](../../examples/espressif/espnn_runner/main/pte_suite.cpp) -is a complete working example, including memory planning and output checks. - -### 3. Build, flash and verify on the board - -The runner example builds and flashes a bundle, then compares the board's -output against the host reference: - -```bash -python examples/espressif/scripts/run_bundle.py --port /dev/ttyACM0 \ - --executorch-build-dir cmake-out-esp-esp32s3 mnv3s.xtp -``` - -``` -ESP32-S3 (QFN56) (revision v0.2): 16 MB flash; building executorch runner for esp32s3 -CHIP target=esp32s3 cores=2 revision=2 cpu_mhz=240 flash_mb=16 psram_kb=8192 ... -ET CASE 1/1 name=mobilenet_v3_small ... delegates=1 delegate_us=... cpu_us=... exact=1 status=PASS -ESPNN RUNNER: PASS -RESULT: PASS (1 cases) -``` - -You can also work in the example directly with `idf.py`. See -[`examples/espressif/espnn_runner/README.md`](../../examples/espressif/espnn_runner/README.md). -[`examples/espressif/camera_stream`](../../examples/espressif/camera_stream/README.md) -runs a model live on camera frames and streams the results to a browser. - ### The original ESP32 -The same flow works on the ESP32 (`idf.py set-target esp32`, or let the -scripts detect the chip). ESP-NN has no SIMD kernels for it and uses portable -C, about 13 million int8 multiply-accumulates per second at 240 MHz, roughly -an order of magnitude slower than the S3. Most ESP32 modules have no PSRAM, so -activations must fit in about 280 KB of internal RAM. Small models fit, such -as keyword spotting, tiny CNNs, and MobileNets at low resolution. Models that -need more RAM fail with a clear allocation error. - -## Benchmarks - -`examples/espressif/scripts/benchmark.py` compiles a fixed set of models. It -groups as many as fit into each flash image, runs them on the connected board -and writes `examples/espressif/benchmarks/.{md,json}`. Every row is -verified against the host reference before it is timed, so a row without -`PASS` is not reported as a result. - -```bash -python examples/espressif/scripts/benchmark.py --port /dev/ttyACM0 \ - --images imagenette2-160 --executorch-build-dir cmake-out-esp-esp32s3 --dashboard -``` - -The results below are at 240 MHz, measured over 10 runs after a warm-up run, -with batch 1. The S3 board was an ESP32-S3 N16R8 (octal PSRAM at 80 MHz, 64 KB -data cache). The ESP32 board was an ESP32-D0WDQ6 (revision 1.0) with 4 MB flash and no PSRAM. - -### ESP32-S3 - -| Model | Input | Path | Ops | Weights | Arena | Latency | On-board check | -|---|---|---|---|---|---|---|---| -| Keyword spotting DS-CNN | 1×1×49×10 | standalone | 12 | 0.03 MB | 16 KB | 16.4 ms | bit-exact | -| Small residual CNN | 1×3×64×64 | standalone | 16 | 0.24 MB | 36 KB | 40.3 ms | bit-exact | -| MobileNetV3-Small | 1×3×96×96 | standalone | 108 | 2.68 MB | 72 KB | 116 ms | bit-exact | -| MobileNetV3-Small | 1×3×224×224 | standalone | 108 | 2.68 MB | 392 KB | 582 ms | bit-exact | -| MobileNetV2 | 1×3×224×224 | standalone | 65 | 3.70 MB | 1.4 MB | 1.61 s | bit-exact | -| MobileNetV3-Large | 1×3×224×224 | standalone | 121 | 5.71 MB | 980 KB | 1.58 s | bit-exact | -| EfficientNet-B0 | 1×3×224×224 | standalone | 190 | 5.65 MB | 2.3 MB | 2.50 s | bit-exact | -| LR-ASPP MobileNetV3 segmentation | 1×3×224×224 | ExecuTorch, 146 delegated / 0 CPU | | 3.43 MB `.pte` | 1.0 MB | 3.58 s | bit-exact (CRC) | -| SSDLite320 MobileNetV3 detection heads | 1×3×320×320 | ExecuTorch, 243 delegated / 0 CPU | | 3.72 MB `.pte` | 2.0 MB | 3.83 s | bit-exact (CRC) | -| Small vision transformer | 1×3×64×64 | ExecuTorch, 11 delegated / 30 CPU | | 0.04 MB `.pte` | 38 KB | 85 ms (delegate 22, CPU 63) | 39.3 dB vs host | - -Arenas that fit internal RAM (the small models) stay there; every 224×224 -model's arena is in PSRAM. The transformer is checked by SQNR because its float CPU -operators (LayerNorm, attention, softmax) round differently from the host. - -### ESP32 (no PSRAM, 4 MB flash) - -Standalone path only. The ExecuTorch path needs ExecuTorch cross-built for -this chip (`build_executorch_esp.sh esp32`), which was not part of this run. - -| Model | Input | Ops | Weights | Arena | Latency | On-board check | -|---|---|---|---|---|---|---| -| Keyword spotting DS-CNN (2.7 MMAC) | 1×1×49×10 | 12 | 0.03 MB | 16 KB | 203.4 ms | bit-exact | -| Small residual CNN (9.3 MMAC) | 1×3×64×64 | 16 | 0.24 MB | 36 KB | 705.0 ms | bit-exact | -| MobileNetV3-Small | 1×3×96×96 | 108 | 2.68 MB | 72 KB | 1440.7 ms | bit-exact | -| MobileNetV3-Small | 1×3×224×224 | 108 | 2.68 MB | 392 KB | does not fit 280 KB RAM | – | -| MobileNetV2 / V3-Large / EfficientNet-B0 at 224 | | | 3.7–5.7 MB | | do not fit 4 MB flash | – | - -Operator suite (`op_suite.xtb`, 20 runs each), all 11 cases bit-exact against -the host reference and ESP-NN's ANSI C kernels: - -| Case | Latency | -|---|---| -| SE block: hardswish LUT, mul, softmax | 6.8 ms | -| Detection heads (reshape/permute/cat) | 9.0 ms | -| Segmentation: dilated conv + transposed conv | 12.0 ms | -| Upsample nearest ×2 | 2.6 ms | -| Upsample bilinear ×2 / align_corners | 2.9 / 2.2 ms | -| Broadcast, pad, split, cat | 2.9 ms | -| Two inputs, two outputs | 1.1 ms | -| Transpose, select, stack | 0.4 ms | - -Full tables with arena sizes and host accuracy are in -[`examples/espressif/benchmarks/`](../../examples/espressif/benchmarks/). +The delegate also runs on the ESP32. ESP-NN has no SIMD kernels for it and +uses portable C, about 13 million int8 multiply-accumulates per second at +240 MHz, roughly an order of magnitude slower than the S3. Most ESP32 modules +have no PSRAM, so activations must fit in about 280 KB of internal RAM. ## Testing @@ -317,15 +185,21 @@ pytest backends/espressif/test The tests export small models for every operator group, lower them, run the XTS3 graphs through the reference interpreter and compare the result against -the quantized PyTorch model. CPU fallback tests run the lowered program in the -ExecuTorch Python runtime. On hardware, -`examples/espressif/scripts/make_test_bundles.py` packs the same models into -`op_suite.xtb` and `pte_suite.xtp` for the runner, and every case must be -bit-exact on the board. +the quantized PyTorch model. `test_xts3_parser_host.py` builds the device +parser for the host and feeds it corrupted graphs. + +End to end, `.ci/scripts/test_espressif.sh` runs the same models through the +ExecuTorch runtime on an emulated ESP32-S3. +`examples/espressif/export_espnn_test_programs.py` exports each one as a +BundleIO program with the host reference's outputs. CI builds the runner with +`-DET_ESPNN_BACKEND=ON` for each program and runs it in `esp-emu`. Fully +delegated programs must match to float precision, which means bit-exact int8 +results. Programs with CPU regions get a tolerance for the portable kernels' +float rounding. The emulator does not implement the S3 SIMD instructions, so +this build uses ESP-NN's ANSI C kernels (`CONFIG_NN_ANSI_C=y`). The code follows the ExecuTorch coding style and passes the repository's -`lintrunner` checks (flake8, ufmt, clang-format, cppcheck, cmake-format and -the license and include linters): +`lintrunner` checks: ```bash lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt index a0209a7be5f..d909069f942 100644 --- a/backends/espressif/espnn_runtime/CMakeLists.txt +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -6,14 +6,16 @@ # # SPDX-License-Identifier: BSD-3-Clause -set(espnn_sources "src/xts3_graph.cpp" "src/int8_interpreter.cpp") -if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) - list(APPEND espnn_sources "src/espnn_backend.cpp") -endif() +# ESP-IDF component providing the ExecuTorch ESP-NN delegate (EspNNBackend). It +# is meant to sit next to examples/espressif/executor_runner, which imports the +# cross-compiled ExecuTorch targets this component links against: build +# examples/espressif/project with -DET_ESPNN_BACKEND=ON. idf_component_register( SRCS - ${espnn_sources} + "src/xts3_graph.cpp" + "src/int8_interpreter.cpp" + "src/espnn_backend.cpp" INCLUDE_DIRS "include" REQUIRES @@ -25,69 +27,16 @@ idf_component_register( WHOLE_ARCHIVE ) -if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) - get_filename_component( - default_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE - ) - set(ESPNN_EXECUTORCH_SOURCE_DIR - "${default_root}" - CACHE PATH "ExecuTorch source tree" - ) - set(executorch_root "${ESPNN_EXECUTORCH_SOURCE_DIR}") - set(ESPNN_EXECUTORCH_BUILD_DIR - "${executorch_root}/cmake-out-esp-${IDF_TARGET}" - CACHE - PATH - "ExecuTorch cross-compiled for this target (examples/espressif/scripts/build_executorch_esp.sh)" - ) - set(et_lib "${ESPNN_EXECUTORCH_BUILD_DIR}/lib") - foreach(required libexecutorch.a libexecutorch_core.a libportable_kernels.a - libkernels_util_all_deps.a - ) - if(NOT EXISTS "${et_lib}/${required}") - message( - FATAL_ERROR - "Missing ${et_lib}/${required}. Build ExecuTorch for ${IDF_TARGET} with " - "examples/espressif/scripts/build_executorch_esp.sh or set " - "-DESPNN_EXECUTORCH_BUILD_DIR=" - ) - endif() - endforeach() - set(et_registries "") - foreach(registry libportable_ops_lib.a libexecutorch_selected_kernels.a - libquantized_ops_lib.a - ) - if(EXISTS "${et_lib}/${registry}") - list(APPEND et_registries "${et_lib}/${registry}") - endif() - endforeach() - set(et_kernels "${et_lib}/libportable_kernels.a") - if(EXISTS "${et_lib}/libquantized_kernels.a") - list(APPEND et_kernels "${et_lib}/libquantized_kernels.a") - endif() - target_include_directories( - ${COMPONENT_LIB} - PUBLIC - "${ESPNN_EXECUTORCH_BUILD_DIR}/include" - "${ESPNN_EXECUTORCH_BUILD_DIR}/include/executorch/runtime/core/portable_type/c10" - "${executorch_root}/.." - ) - target_compile_definitions( - ${COMPONENT_LIB} PUBLIC C10_USING_CUSTOM_GENERATED_MACROS - ) - # Kernel and backend registration happens in static initializers, so these - # archives are linked whole. One flag per archive: CMake deduplicates a bare - # -Wl,--whole-archive pair. - set(et_whole_archives "") - foreach(archive ${et_registries} "${et_lib}/libexecutorch.a") - list(APPEND et_whole_archives - "-Wl,--whole-archive,${archive},--no-whole-archive" - ) - endforeach() - target_link_libraries( - ${COMPONENT_LIB} - INTERFACE ${et_whole_archives} ${et_kernels} - "${et_lib}/libkernels_util_all_deps.a" - "${et_lib}/libexecutorch_core.a" - ) -endif() +get_filename_component( + espnn_executorch_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE +) +target_include_directories( + ${COMPONENT_LIB} + PRIVATE "${espnn_executorch_root}/.." + "${espnn_executorch_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + ${COMPONENT_LIB} PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +# Imported by executor_runner's find_package(executorch) as a global target. +target_link_libraries(${COMPONENT_LIB} PUBLIC executorch_core) diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig index c9cc88b65f1..8fa51b176b6 100644 --- a/backends/espressif/espnn_runtime/Kconfig +++ b/backends/espressif/espnn_runtime/Kconfig @@ -8,27 +8,17 @@ menu "ExecuTorch ESP-NN backend" - config ESPNN_EXECUTORCH_DELEGATE - bool "Build the ExecuTorch delegate (EspNNBackend) and link the ExecuTorch runtime" - default n - help - Needs ExecuTorch cross-compiled for this chip, see - examples/espressif/scripts/build_executorch_esp.sh. Without it the - component still provides the standalone XTS3 int8 interpreter. - config ESPNN_DELEGATE_CONSTANTS_TO_PSRAM bool "Copy delegate weights from flash into PSRAM when they fit" default y - depends on ESPNN_EXECUTORCH_DELEGATE && SPIRAM + depends on SPIRAM config ESPNN_DELEGATE_DUAL_CORE bool "Split ESP-NN kernels across both cores inside delegates" default n - depends on ESPNN_EXECUTORCH_DELEGATE config ESPNN_DELEGATE_PROFILE bool "Print a per-operator profile after each delegate's first run" default n - depends on ESPNN_EXECUTORCH_DELEGATE endmenu diff --git a/examples/espressif/.gitignore b/examples/espressif/.gitignore index 771b12ba266..f17ff0c9643 100644 --- a/examples/espressif/.gitignore +++ b/examples/espressif/.gitignore @@ -5,6 +5,4 @@ dependencies.lock sdkconfig sdkconfig.old __pycache__/ -espnn_runner/bundles/ camera_stream/models/ -benchmarks/work/ diff --git a/examples/espressif/aot_espnn_compiler.py b/examples/espressif/aot_espnn_compiler.py deleted file mode 100644 index ccd7aac4112..00000000000 --- a/examples/espressif/aot_espnn_compiler.py +++ /dev/null @@ -1,515 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Quantize a PyTorch model to int8 and lower it for Espressif chips with the -ESP-NN backend, producing a bundle that examples/espressif/espnn_runner can -flash and verify on the board. - -Modes: - executorch (default) ExecuTorch program (.pte): int8 regions run in the - EspNNBackend delegate, anything else on the portable - CPU kernels. Any number of inputs and outputs. - standalone The whole model as one XTS3 int8 graph for the - espnn_runtime interpreter without the ExecuTorch - runtime (smallest firmware). Every operator must lower. - -Examples: - python examples/espressif/aot_espnn_compiler.py --model mobilenet_v2 \ - --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val --output mv2.xtp - python examples/espressif/aot_espnn_compiler.py --model my_models.py:make_model \ - --input-shape 1,3,96,96 --input-shape 1,10 --mode executorch --output mine.xtp - python examples/espressif/aot_espnn_compiler.py --model mobilenet_v3_small --mode standalone \ - --input-shape 1,3,96,96 --output mv3s_96.xtb -""" - -from __future__ import annotations - -import argparse -import copy -import importlib.util -import json -import math -import random -import sys -import time -from collections import Counter -from pathlib import Path - -import numpy as np -import torch - -from executorch.backends.espressif.quantizer import EspNNQuantizer -from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from espnn_bundle import sqnr_db, write_bundle, xte1_vectors, xtv2_vectors # noqa: E402 - -IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} -IMAGENETTE_WNID_TO_IMAGENET = { - "n01440764": 0, - "n02102040": 217, - "n02979186": 482, - "n03000684": 491, - "n03028079": 497, - "n03394916": 566, - "n03417042": 569, - "n03425413": 571, - "n03445777": 574, - "n03888257": 701, -} - - -def load_model(spec: str, weights: str) -> torch.nn.Module: - if ":" in spec and spec.split(":", 1)[0].endswith(".py"): - path, factory = spec.split(":", 1) - module_spec = importlib.util.spec_from_file_location("espnn_user_model", path) - if module_spec is None or module_spec.loader is None: - raise ValueError(f"Unable to import {path}") - module = importlib.util.module_from_spec(module_spec) - sys.modules[module_spec.name] = module - module_spec.loader.exec_module(module) - return getattr(module, factory)() - from torchvision import models - - builder = getattr(models, spec, None) - if builder is None: - raise ValueError( - f"Unknown torchvision model {spec}; use path/to/file.py:factory" - ) - return builder(weights=None if weights == "none" else weights) - - -class ImageSource: - def __init__(self, shape, resize: int, mean, std, label_map): - from torchvision import transforms - - height, width = shape[2], shape[3] - self.transform = transforms.Compose( - [ - transforms.Resize(resize or min(height, width)), - transforms.CenterCrop((height, width)), - transforms.ToTensor(), - transforms.Normalize(mean, std), - ] - ) - self.label_map = dict(IMAGENETTE_WNID_TO_IMAGENET) - if label_map: - self.label_map.update(json.loads(Path(label_map).read_text())) - - def load(self, directory: Path, count: int, seed: int): - from PIL import Image - - files = sorted( - p for p in Path(directory).rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES - ) - if not files: - raise ValueError(f"No images found under {directory}") - random.Random(seed).shuffle(files) - return [ - ( - (self.transform(Image.open(path).convert("RGB")).unsqueeze(0),), - int(self.label_map.get(path.parent.name, -1)), - ) - for path in files[:count] - ] - - -def load_npy(path: Path, count: int, shape): - array = np.load(path) - if tuple(array.shape[1:]) != tuple(shape[1:]): - raise ValueError( - f"{path} has sample shape {array.shape[1:]}, expected {shape[1:]}" - ) - return [ - ((torch.from_numpy(array[i : i + 1]).float(),), -1) - for i in range(min(count, len(array))) - ] - - -def make_data(args, shapes): - image_input = len(shapes) == 1 and len(shapes[0]) == 4 and shapes[0][1] == 3 - source = None - if image_input and (args.calib_dir or args.eval_dir): - source = ImageSource( - shapes[0], args.resize, args.mean, args.std, args.label_map - ) - generator = torch.Generator().manual_seed(args.seed) - - def noise(count): - return [ - (tuple(torch.randn(s, generator=generator) for s in shapes), -1) - for _ in range(count) - ] - - if args.calib_npy and len(shapes) == 1: - calibration = load_npy(args.calib_npy, args.calib_samples, shapes[0]) - elif source is not None and args.calib_dir: - calibration = source.load(args.calib_dir, args.calib_samples, args.seed) - else: - print( - " NOTE: no calibration data given; using Gaussian noise (accuracy will be poor)" - ) - calibration = noise(args.calib_samples) - if args.eval_npy and len(shapes) == 1: - evaluation = load_npy(args.eval_npy, args.eval_samples, shapes[0]) - elif source is not None and args.eval_dir: - evaluation = source.load(args.eval_dir, args.eval_samples, args.seed + 1) - else: - evaluation = calibration[: args.eval_samples] - return calibration, evaluation - - -def qat_finetune(model, calibration, args): - """Quantization-aware fine-tuning by KL distillation from the float model (no labels).""" - import torch.nn.functional as F - from torchao.quantization.pt2e import ( - disable_observer, - move_exported_model_to_eval, - move_exported_model_to_train, - ) - from torchao.quantization.pt2e.quantize_pt2e import prepare_qat_pt2e - - data = torch.cat([inputs[0] for inputs, _ in calibration]) - batch = min(args.qat_batch, len(data)) - student = copy.deepcopy(model) - exported = torch.export.export( - student, - (data[:2],), - dynamic_shapes=({0: torch.export.Dim("batch", min=2, max=1024)},), - strict=True, - ).module() - prepared = prepare_qat_pt2e(exported, EspNNQuantizer(is_qat=True)) - for parameter in prepared.parameters(): - if parameter.is_floating_point(): - parameter.requires_grad_(True) - parameters = [p for p in prepared.parameters() if p.requires_grad] - total_steps = max(1, int(round(args.qat_epochs * max(1, len(data) // batch)))) - optimizer = torch.optim.SGD(parameters, lr=args.qat_lr, momentum=0.9) - scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, total_steps) - generator = torch.Generator().manual_seed(args.seed) - move_exported_model_to_train(prepared) - with torch.no_grad(): - for start in range(0, min(len(data), 16 * batch), batch): - prepared(data[start : start + batch]) - with torch.enable_grad(): - for step in range(total_steps): - if step == int(total_steps * 0.6): - prepared.apply(disable_observer) - inputs = data[torch.randint(0, len(data), (batch,), generator=generator)] - if bool(torch.rand(1, generator=generator) < 0.5): - inputs = inputs.flip(3) - with torch.no_grad(): - teacher = F.softmax(model(inputs), dim=1) - loss = F.kl_div( - F.log_softmax(prepared(inputs), dim=1), teacher, reduction="batchmean" - ) - optimizer.zero_grad() - loss.backward() - optimizer.step() - scheduler.step() - if step % max(1, total_steps // 10) == 0 or step + 1 == total_steps: - print(f" QAT step {step + 1}/{total_steps} loss {loss.item():.4f}") - move_exported_model_to_eval(prepared) - for parameter in prepared.parameters(): - parameter.requires_grad_(False) - return prepared - - -def quantize(model, calibration, args): - example = calibration[0][0] - if args.qat_epochs > 0: - if len(example) != 1: - raise SystemExit("QAT is implemented for single-input models") - print(f"[2/5] PT2E quantization-aware fine-tuning: {args.qat_epochs} epochs") - prepared = qat_finetune(model, calibration, args) - else: - print(f"[2/5] PT2E quantization with {len(calibration)} calibration samples") - prepared = prepare_pt2e( - torch.export.export(model, example, strict=True).module(), EspNNQuantizer() - ) - for inputs, _ in calibration: - prepared(*inputs) - quantized = convert_pt2e(prepared) - return quantized, torch.export.export(quantized, example, strict=True) - - -def as_list(value): - return list(value) if isinstance(value, (tuple, list)) else [value] - - -def evaluate(model, quantized, device_fn, evaluation): - totals = Counter() - worst_float = worst_pt2e = math.inf - for inputs, label in evaluation: - float_out = as_list(model(*inputs)) - pt2e_out = as_list(quantized(*inputs)) - device_out = as_list(device_fn(inputs)) - for f, q, d in zip(float_out, pt2e_out, device_out): - worst_float = min(worst_float, sqnr_db(f, d.to(torch.float32))) - worst_pt2e = min(worst_pt2e, sqnr_db(q, d.to(torch.float32))) - if float_out[0].dim() == 2: - float_top1 = int(float_out[0].argmax()) - device_top1 = int(device_out[0].to(torch.float32).argmax()) - totals["samples"] += 1 - totals["top1_agree_with_float"] += int(float_top1 == device_top1) - if label >= 0: - totals["labelled"] += 1 - totals["float_correct"] += int(float_top1 == label) - totals["device_correct"] += int(device_top1 == label) - print( - f" worst output SQNR: device vs float {worst_float:.2f} dB, " - f"device vs PT2E int8 {worst_pt2e:.2f} dB" - ) - if totals["samples"]: - line = f" top-1 agreement with float {totals['top1_agree_with_float']}/{totals['samples']}" - if totals["labelled"]: - line += ( - f"; accuracy float {totals['float_correct']}/{totals['labelled']}, " - f"device {totals['device_correct']}/{totals['labelled']}" - ) - print(line) - return worst_float, worst_pt2e, dict(totals) - - -def compile_standalone(args, model, quantized, program, evaluation): - from executorch.backends.espressif.reference import esp_nn_reference as reference - from executorch.backends.espressif.serialization import ( - build_graph, - parse_graph, - serialize_graph, - ) - - print("[3/5] Lowering to one XTS3 int8 graph (standalone espnn_runtime)") - blob = serialize_graph(build_graph(program)) - graph = parse_graph(blob) - interpreter = reference.ReferenceInterpreter(graph) - ops = Counter(op.opcode.name for op in graph.operations) - arena, _ = reference.plan_activation_arena(graph) - print( - f" {len(graph.operations)} operations {dict(ops)}; blob {len(blob)} bytes; " - f"activation arena {arena} bytes" - ) - - def device_fn(inputs): - device_inputs = [ - reference.to_device_layout( - graph.tensors[i], reference.quantize(graph.tensors[i], x.numpy()) - ) - for i, x in zip(interpreter.inputs, inputs) - ] - outputs = interpreter.run_outputs(device_inputs) - return [ - torch.from_numpy( - reference.dequantize( - graph.tensors[i], reference.to_logical_layout(graph.tensors[i], out) - ) - ) - for i, out in zip(interpreter.outputs, outputs) - ] - - print( - f"[4/5] Evaluating {len(evaluation)} samples: float vs PT2E int8 vs bit-exact device model" - ) - worst_float, worst_pt2e, totals = evaluate(model, quantized, device_fn, evaluation) - samples = [ - [x.numpy() for x in inputs] for inputs, _ in evaluation[: args.vector_samples] - ] - vectors = xtv2_vectors(graph, samples) - report = { - "mode": "standalone", - "operations": dict(ops), - "blob_bytes": len(blob), - "activation_arena_bytes": arena, - "inputs": [list(graph.tensors[i].shape) for i in graph.inputs], - "outputs": [list(graph.tensors[i].shape) for i in graph.outputs], - } - return b"XTB1", blob, vectors, report, worst_float, worst_pt2e, totals - - -def compile_executorch(args, model, quantized, program, evaluation): - from executorch.backends.espressif import to_espnn_edge - from executorch.backends.espressif.reference import ProgramEmulator - from executorch.exir.lowered_backend_module import LoweredBackendModule - - print("[3/5] Lowering through ExecuTorch: EspNN delegates + portable CPU fallback") - edge, partition_report = to_espnn_edge( - program, allow_cpu_fallback=not args.no_cpu_fallback - ) - exported = edge.exported_program() - delegates = [ - m - for _, m in exported.graph_module.named_children() - if isinstance(m, LoweredBackendModule) - ] - pte = edge.to_executorch().buffer - cpu_ops = partition_report["cpu_ops"] - print( - f" {len(delegates)} delegate(s), {partition_report['delegated_ops']} delegated operators, " - f"{len(cpu_ops)} operators on the CPU; .pte {len(pte)} bytes" - ) - for target, count in Counter(op["target"] for op in cpu_ops).most_common(12): - reason = next(op["reason"] for op in cpu_ops if op["target"] == target) - print(f" CPU: {target} x{count} ({reason})") - emulator = ProgramEmulator(exported) - print( - f"[4/5] Evaluating {len(evaluation)} samples: float vs PT2E int8 vs emulated device program" - ) - worst_float, worst_pt2e, totals = evaluate( - model, quantized, lambda inputs: as_list(emulator(*inputs)), evaluation - ) - rows = [ - (list(inputs), as_list(emulator(*inputs))) - for inputs, _ in evaluation[: args.vector_samples] - ] - exact = not cpu_ops - vectors = xte1_vectors( - rows, exact, args.device_min_sqnr, crc=exact and args.crc_outputs - ) - if args.pte_output: - args.pte_output.write_bytes(pte) - report = { - "mode": "executorch", - "pte_bytes": len(pte), - "delegates": len(delegates), - "partitions": partition_report["partitions"], - "delegated_ops": partition_report["delegated_ops"], - "cpu_ops": cpu_ops, - "exact_required": exact, - "min_sqnr_db": args.device_min_sqnr, - "inputs": [list(t.shape) for t in rows[0][0]], - "outputs": [list(t.shape) for t in rows[0][1]], - } - return b"XTP1", pte, vectors, report, worst_float, worst_pt2e, totals - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--model", - required=True, - help="torchvision model name or path/to/file.py:factory", - ) - parser.add_argument( - "--weights", default="DEFAULT", help="torchvision weights name, or 'none'" - ) - parser.add_argument( - "--input-shape", - action="append", - help="input shape such as 1,3,224,224; repeat for several inputs", - ) - parser.add_argument( - "--mode", choices=("executorch", "standalone"), default="executorch" - ) - parser.add_argument( - "--output", type=Path, required=True, help="bundle to write (.xtp or .xtb)" - ) - parser.add_argument( - "--name", help="entry name inside the bundle (default: model name)" - ) - parser.add_argument( - "--pte-output", type=Path, help="executorch mode: also write the plain .pte" - ) - parser.add_argument("--calib-dir", type=Path) - parser.add_argument("--calib-npy", type=Path) - parser.add_argument("--calib-samples", type=int, default=128) - parser.add_argument("--eval-dir", type=Path) - parser.add_argument("--eval-npy", type=Path) - parser.add_argument("--eval-samples", type=int, default=32) - parser.add_argument( - "--vector-samples", - type=int, - default=4, - help="samples embedded in the bundle for on-board checking", - ) - parser.add_argument("--resize", type=int, default=256) - parser.add_argument("--mean", type=float, nargs=3, default=[0.485, 0.456, 0.406]) - parser.add_argument("--std", type=float, nargs=3, default=[0.229, 0.224, 0.225]) - parser.add_argument( - "--label-map", help="JSON mapping image folder names to class indices" - ) - parser.add_argument("--qat-epochs", type=float, default=0.0) - parser.add_argument("--qat-lr", type=float, default=2e-4) - parser.add_argument("--qat-batch", type=int, default=16) - parser.add_argument( - "--no-cpu-fallback", - action="store_true", - help="executorch mode: fail if any operator would run on the CPU", - ) - parser.add_argument( - "--device-min-sqnr", - type=float, - default=30.0, - help="on-board pass threshold (dB vs host) for programs with CPU operators", - ) - parser.add_argument( - "--crc-outputs", - action="store_true", - help="store CRC32 instead of full expected outputs (large outputs)", - ) - parser.add_argument( - "--min-sqnr", - type=float, - default=-math.inf, - help="fail when the device model's SQNR against float is lower", - ) - parser.add_argument("--seed", type=int, default=0) - args = parser.parse_args() - - torch.set_grad_enabled(False) - torch.manual_seed(args.seed) - started = time.time() - shapes = [ - tuple(int(v) for v in s.split(",")) - for s in (args.input_shape or ["1,3,224,224"]) - ] - print(f"[1/5] Loading {args.model}") - model = load_model(args.model, args.weights).eval() - calibration, evaluation = make_data(args, shapes) - quantized, program = quantize(model, calibration, args) - compile_fn = compile_standalone if args.mode == "standalone" else compile_executorch - magic, blob, vectors, report, worst_float, worst_pt2e, totals = compile_fn( - args, model, quantized, program, evaluation - ) - print("[5/5] Writing artifacts") - name = args.name or Path(args.model.split(":")[-1]).stem - size = write_bundle(args.output, magic, [(name, blob, vectors)]) - report.update( - { - "name": name, - "model": args.model, - "bundle": str(args.output), - "bundle_bytes": size, - "calibration_samples": len(calibration), - "qat_epochs": args.qat_epochs, - "sqnr_vs_float_db": worst_float, - "sqnr_vs_pt2e_db": worst_pt2e, - "evaluation": totals, - } - ) - args.output.with_suffix(".json").write_text(json.dumps([report], indent=2)) - print( - f" bundle {args.output} ({size} bytes); report {args.output.with_suffix('.json')}" - ) - print( - f" embed it: CONFIG_ESPNN_RUNNER_BUNDLE (espnn_runner) or CONFIG_CAMERA_STREAM_MODEL (camera_stream) = {args.output.resolve()}" - ) - print(f" elapsed {time.time() - started:.1f} s") - if worst_float < args.min_sqnr: - print( - f"ESPNN COMPILE: FAIL (SQNR {worst_float:.2f} dB below {args.min_sqnr:.2f} dB)" - ) - return 1 - print("ESPNN COMPILE: PASS") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/espressif/benchmarks/esp32.json b/examples/espressif/benchmarks/esp32.json deleted file mode 100644 index e3d083d2593..00000000000 --- a/examples/espressif/benchmarks/esp32.json +++ /dev/null @@ -1,186 +0,0 @@ -{ - "target": "esp32", - "chip": { - "target": "esp32", - "cores": "2", - "revision": "100", - "cpu_mhz": "240", - "flash_mb": "4", - "psram_kb": "0", - "internal_free_kb": "280", - "idf": "v5.5.5" - }, - "esptool": "ESP32-D0WDQ6 (revision v1.0)", - "date": "2026-10-18", - "iterations": 10, - "rows": [ - { - "name": "ds_cnn_kws", - "description": "Keyword spotting DS-CNN, 49x10 MFCC", - "input": "1,1,49,10", - "path": "standalone XTS3", - "operators": "12 int8", - "size": "0.03 MB", - "accuracy": "top-1 agrees with float 32/32", - "latency": "203.4 ms (203.4)", - "fps": "4.92", - "arena": "16 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "tiny_resnet_64", - "description": "Small residual CNN, 64x64", - "input": "1,3,64,64", - "path": "standalone XTS3", - "operators": "16 int8", - "size": "0.24 MB", - "accuracy": "top-1 agrees with float 32/32", - "latency": "705.0 ms (705.0)", - "fps": "1.42", - "arena": "36 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v3_small_96", - "description": "MobileNetV3-Small, ImageNet, 96x96", - "input": "1,3,96,96", - "path": "standalone XTS3", - "operators": "108 int8", - "size": "2.68 MB", - "accuracy": "top-1 0/32 (float 6), agrees with float 0/32", - "latency": "1440.7 ms (1440.7)", - "fps": "0.69", - "arena": "72 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v3_small_224", - "description": "MobileNetV3-Small, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "108 int8", - "size": "2.68 MB", - "accuracy": "top-1 6/32 (float 21), agrees with float 4/32", - "latency": "-", - "fps": "-", - "arena": "392 KB", - "check": "does not fit RAM (280 KB free, no PSRAM)" - }, - { - "name": "mobilenet_v2_224", - "description": "MobileNetV2, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "65 int8", - "size": "3.70 MB", - "accuracy": "top-1 20/32 (float 20), agrees with float 29/32", - "latency": "-", - "fps": "-", - "arena": "-", - "check": "does not fit 4 MB flash" - }, - { - "name": "mobilenet_v3_large_224", - "description": "MobileNetV3-Large, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "121 int8", - "size": "5.71 MB", - "accuracy": "top-1 24/32 (float 23), agrees with float 27/32", - "latency": "-", - "fps": "-", - "arena": "-", - "check": "does not fit 4 MB flash" - }, - { - "name": "efficientnet_b0_224", - "description": "EfficientNet-B0, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "190 int8", - "size": "5.65 MB", - "accuracy": "top-1 19/32 (float 23), agrees with float 23/32", - "latency": "-", - "fps": "-", - "arena": "-", - "check": "does not fit 4 MB flash" - } - ], - "raw_cases": { - "ds_cnn_kws": { - "name": "ds_cnn_kws", - "ops": "12", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "203431", - "min_us": "203419", - "max_us": "203490", - "iterations": "10", - "arena": "16000", - "arena_internal": "1", - "constants_in_ram": "0", - "blob": "31024", - "status": "PASS", - "kind": "standalone" - }, - "tiny_resnet_64": { - "name": "tiny_resnet_64", - "ops": "16", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "705045", - "min_us": "705029", - "max_us": "705066", - "iterations": "10", - "arena": "36864", - "arena_internal": "1", - "constants_in_ram": "0", - "blob": "242808", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v3_small_96": { - "name": "mobilenet_v3_small_96", - "ops": "108", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "1440734", - "min_us": "1440709", - "max_us": "1440760", - "iterations": "10", - "arena": "73728", - "arena_internal": "1", - "constants_in_ram": "0", - "blob": "2678784", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v3_small_224": { - "name": "mobilenet_v3_small_224", - "status": "FAIL", - "reason": "load", - "kind": "standalone", - "error": "unable to allocate 401408-byte activation arena" - } - }, - "failures": { - "mobilenet_v2_224": "does not fit 4 MB flash", - "efficientnet_b0_224": "does not fit 4 MB flash", - "mobilenet_v3_large_224": "does not fit 4 MB flash" - } -} \ No newline at end of file diff --git a/examples/espressif/benchmarks/esp32.md b/examples/espressif/benchmarks/esp32.md deleted file mode 100644 index 913fb7f5311..00000000000 --- a/examples/espressif/benchmarks/esp32.md +++ /dev/null @@ -1,17 +0,0 @@ -# ESP-NN backend benchmarks: esp32 - -Board: esp32, 2 cores at 240 MHz, 4 MB flash, 0 KB PSRAM, ESP-IDF v5.5.5. - -| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy | -|---|---|---|---|---|---|---|---|---|---| -| Keyword spotting DS-CNN, 49x10 MFCC | 1,1,49,10 | standalone XTS3 | 12 int8 | 0.03 MB | 16 KB | 203.4 ms (203.4) | 4.92 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | -| Small residual CNN, 64x64 | 1,3,64,64 | standalone XTS3 | 16 int8 | 0.24 MB | 36 KB | 705.0 ms (705.0) | 1.42 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | -| MobileNetV3-Small, ImageNet, 96x96 | 1,3,96,96 | standalone XTS3 | 108 int8 | 2.68 MB | 72 KB | 1440.7 ms (1440.7) | 0.69 | bit-exact 2/2, ANSI too | top-1 0/32 (float 6), agrees with float 0/32 | -| MobileNetV3-Small, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 108 int8 | 2.68 MB | 392 KB | - | - | does not fit RAM (280 KB free, no PSRAM) | top-1 6/32 (float 21), agrees with float 4/32 | -| MobileNetV2, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 65 int8 | 3.70 MB | - | - | - | does not fit 4 MB flash | top-1 20/32 (float 20), agrees with float 29/32 | -| MobileNetV3-Large, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 121 int8 | 5.71 MB | - | - | - | does not fit 4 MB flash | top-1 24/32 (float 23), agrees with float 27/32 | -| EfficientNet-B0, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 190 int8 | 5.65 MB | - | - | - | does not fit 4 MB flash | top-1 19/32 (float 23), agrees with float 23/32 | - -Latency is the model invocation only (no input preprocessing), timed over repeated runs after a bit-exact (or SQNR) check against the host reference. - -Host accuracy is the int8 model on the host (the board's output is bit-identical) after post-training quantization with 64 calibration images and no QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with random weights report top-1 agreement or SQNR against the float model instead. diff --git a/examples/espressif/benchmarks/esp32s3.json b/examples/espressif/benchmarks/esp32s3.json deleted file mode 100644 index bad7a12f31f..00000000000 --- a/examples/espressif/benchmarks/esp32s3.json +++ /dev/null @@ -1,370 +0,0 @@ -{ - "target": "esp32s3", - "chip": { - "target": "esp32s3", - "cores": "2", - "revision": "2", - "cpu_mhz": "240", - "flash_mb": "16", - "psram_kb": "8192", - "internal_free_kb": "252", - "idf": "v5.5.5" - }, - "esptool": "ESP32-S3 (QFN56) (revision v0.2)", - "date": "2026-10-18", - "iterations": 10, - "rows": [ - { - "name": "ds_cnn_kws", - "description": "Keyword spotting DS-CNN, 49x10 MFCC", - "input": "1,1,49,10", - "path": "standalone XTS3", - "operators": "12 int8", - "size": "0.03 MB", - "accuracy": "top-1 agrees with float 32/32", - "latency": "16.4 ms (16.4)", - "fps": "60.80", - "arena": "16 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "tiny_resnet_64", - "description": "Small residual CNN, 64x64", - "input": "1,3,64,64", - "path": "standalone XTS3", - "operators": "16 int8", - "size": "0.24 MB", - "accuracy": "top-1 agrees with float 32/32", - "latency": "40.3 ms (40.3)", - "fps": "24.80", - "arena": "36 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v3_small_96", - "description": "MobileNetV3-Small, ImageNet, 96x96", - "input": "1,3,96,96", - "path": "standalone XTS3", - "operators": "108 int8", - "size": "2.68 MB", - "accuracy": "top-1 0/32 (float 6), agrees with float 0/32", - "latency": "116.2 ms (116.2)", - "fps": "8.61", - "arena": "72 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v3_small_224", - "description": "MobileNetV3-Small, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "108 int8", - "size": "2.68 MB", - "accuracy": "top-1 6/32 (float 21), agrees with float 4/32", - "latency": "581.7 ms (581.7)", - "fps": "1.72", - "arena": "392 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v2_224", - "description": "MobileNetV2, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "65 int8", - "size": "3.70 MB", - "accuracy": "top-1 20/32 (float 20), agrees with float 29/32", - "latency": "1609.3 ms (1609.3)", - "fps": "0.62", - "arena": "1470 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "mobilenet_v3_large_224", - "description": "MobileNetV3-Large, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "121 int8", - "size": "5.71 MB", - "accuracy": "top-1 24/32 (float 23), agrees with float 27/32", - "latency": "1581.6 ms (1581.6)", - "fps": "0.63", - "arena": "980 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "efficientnet_b0_224", - "description": "EfficientNet-B0, ImageNet, 224x224", - "input": "1,3,224,224", - "path": "standalone XTS3", - "operators": "190 int8", - "size": "5.65 MB", - "accuracy": "top-1 19/32 (float 23), agrees with float 23/32", - "latency": "2498.3 ms (2498.3)", - "fps": "0.40", - "arena": "2352 KB", - "check": "bit-exact 2/2, ANSI too" - }, - { - "name": "lraspp_mobilenet_v3_seg", - "description": "LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224", - "input": "1,3,224,224", - "path": "ExecuTorch .pte", - "operators": "146 delegated, 0 CPU", - "size": "3.43 MB", - "accuracy": "6.1 dB vs float", - "latency": "3580.4 ms (3579.1); delegate 3580.5, CPU 0.0", - "fps": "0.28", - "arena": "1045 KB", - "check": "bit-exact" - }, - { - "name": "ssdlite320_mobilenet_v3_det", - "description": "SSDLite320 MobileNetV3 detection heads, COCO, 320x320", - "input": "1,3,320,320", - "path": "ExecuTorch .pte", - "operators": "243 delegated, 0 CPU", - "size": "3.72 MB", - "accuracy": "7.4 dB vs float", - "latency": "3830.3 ms (3830.0); delegate 3830.4, CPU 0.0", - "fps": "0.26", - "arena": "2000 KB", - "check": "bit-exact" - }, - { - "name": "tiny_transformer_fallback", - "description": "Small vision transformer: LayerNorm and attention on the CPU fallback", - "input": "1,3,64,64", - "path": "ExecuTorch .pte", - "operators": "11 delegated, 30 CPU", - "size": "0.04 MB", - "accuracy": "top-1 agrees with float 31/32", - "latency": "85.0 ms (84.9); delegate 22.1, CPU 62.8", - "fps": "11.77", - "arena": "38 KB", - "check": "39.3 dB vs host" - } - ], - "raw_cases": { - "ds_cnn_kws": { - "name": "ds_cnn_kws", - "ops": "12", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "16448", - "min_us": "16444", - "max_us": "16451", - "iterations": "10", - "arena": "16000", - "arena_internal": "1", - "constants_in_ram": "1", - "blob": "31024", - "status": "PASS", - "kind": "standalone" - }, - "tiny_resnet_64": { - "name": "tiny_resnet_64", - "ops": "16", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "40319", - "min_us": "40316", - "max_us": "40324", - "iterations": "10", - "arena": "36864", - "arena_internal": "1", - "constants_in_ram": "1", - "blob": "242808", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v3_small_96": { - "name": "mobilenet_v3_small_96", - "ops": "108", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "116161", - "min_us": "116159", - "max_us": "116166", - "iterations": "10", - "arena": "73728", - "arena_internal": "1", - "constants_in_ram": "1", - "blob": "2678784", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v3_small_224": { - "name": "mobilenet_v3_small_224", - "ops": "108", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "581747", - "min_us": "581736", - "max_us": "581804", - "iterations": "10", - "arena": "401408", - "arena_internal": "0", - "constants_in_ram": "1", - "blob": "2678784", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v2_224": { - "name": "mobilenet_v2_224", - "ops": "65", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "1609305", - "min_us": "1609302", - "max_us": "1609334", - "iterations": "10", - "arena": "1505280", - "arena_internal": "0", - "constants_in_ram": "1", - "blob": "3696944", - "status": "PASS", - "kind": "standalone" - }, - "efficientnet_b0_224": { - "name": "efficientnet_b0_224", - "ops": "190", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "2498306", - "min_us": "2498284", - "max_us": "2498400", - "iterations": "10", - "arena": "2408448", - "arena_internal": "0", - "constants_in_ram": "0", - "blob": "5652624", - "status": "PASS", - "kind": "standalone" - }, - "mobilenet_v3_large_224": { - "name": "mobilenet_v3_large_224", - "ops": "121", - "inputs": "1", - "outputs": "1", - "samples": "2", - "exact": "2", - "mismatched": "0", - "worst": "0", - "ansi": "1", - "us": "1581600", - "min_us": "1581592", - "max_us": "1581606", - "iterations": "10", - "arena": "1003520", - "arena_internal": "0", - "constants_in_ram": "1", - "blob": "5709680", - "status": "PASS", - "kind": "standalone" - }, - "tiny_transformer_fallback": { - "name": "tiny_transformer_fallback", - "inputs": "1", - "outputs": "1", - "samples": "2", - "delegates": "5", - "delegate_calls": "5", - "us": "84957", - "min_us": "84865", - "max_us": "85108", - "iterations": "10", - "delegate_us": "22127", - "cpu_us": "62830", - "sqnr_db": "39.3", - "max_abs": "0.00264995", - "mismatched": "5", - "exact_required": "0", - "min_sqnr": "30.0", - "load_us": "7186", - "planned": "90112", - "arena": "38960", - "status": "PASS", - "kind": "executorch" - }, - "lraspp_mobilenet_v3_seg": { - "name": "lraspp_mobilenet_v3_seg", - "inputs": "1", - "outputs": "1", - "samples": "2", - "delegates": "1", - "delegate_calls": "1", - "us": "3580418", - "min_us": "3579074", - "max_us": "3584194", - "iterations": "10", - "delegate_us": "3580532", - "cpu_us": "-114", - "sqnr_db": "999.0", - "max_abs": "0", - "mismatched": "0", - "exact_required": "1", - "min_sqnr": "30.0", - "load_us": "47890", - "planned": "4816896", - "arena": "1070160", - "status": "PASS", - "kind": "executorch" - }, - "ssdlite320_mobilenet_v3_det": { - "name": "ssdlite320_mobilenet_v3_det", - "inputs": "1", - "outputs": "2", - "samples": "2", - "delegates": "1", - "delegate_calls": "1", - "us": "3830285", - "min_us": "3829961", - "max_us": "3832876", - "iterations": "10", - "delegate_us": "3830380", - "cpu_us": "-95", - "sqnr_db": "999.0", - "max_abs": "0", - "mismatched": "0", - "exact_required": "1", - "min_sqnr": "30.0", - "load_us": "80055", - "planned": "2457728", - "arena": "2048000", - "status": "PASS", - "kind": "executorch" - } - }, - "failures": {} -} \ No newline at end of file diff --git a/examples/espressif/benchmarks/esp32s3.md b/examples/espressif/benchmarks/esp32s3.md deleted file mode 100644 index bdf17b92a0f..00000000000 --- a/examples/espressif/benchmarks/esp32s3.md +++ /dev/null @@ -1,20 +0,0 @@ -# ESP-NN backend benchmarks: esp32s3 - -Board: esp32s3, 2 cores at 240 MHz, 16 MB flash, 8192 KB PSRAM, ESP-IDF v5.5.5. - -| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy | -|---|---|---|---|---|---|---|---|---|---| -| Keyword spotting DS-CNN, 49x10 MFCC | 1,1,49,10 | standalone XTS3 | 12 int8 | 0.03 MB | 16 KB | 16.4 ms (16.4) | 60.80 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | -| Small residual CNN, 64x64 | 1,3,64,64 | standalone XTS3 | 16 int8 | 0.24 MB | 36 KB | 40.3 ms (40.3) | 24.80 | bit-exact 2/2, ANSI too | top-1 agrees with float 32/32 | -| MobileNetV3-Small, ImageNet, 96x96 | 1,3,96,96 | standalone XTS3 | 108 int8 | 2.68 MB | 72 KB | 116.2 ms (116.2) | 8.61 | bit-exact 2/2, ANSI too | top-1 0/32 (float 6), agrees with float 0/32 | -| MobileNetV3-Small, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 108 int8 | 2.68 MB | 392 KB | 581.7 ms (581.7) | 1.72 | bit-exact 2/2, ANSI too | top-1 6/32 (float 21), agrees with float 4/32 | -| MobileNetV2, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 65 int8 | 3.70 MB | 1470 KB | 1609.3 ms (1609.3) | 0.62 | bit-exact 2/2, ANSI too | top-1 20/32 (float 20), agrees with float 29/32 | -| MobileNetV3-Large, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 121 int8 | 5.71 MB | 980 KB | 1581.6 ms (1581.6) | 0.63 | bit-exact 2/2, ANSI too | top-1 24/32 (float 23), agrees with float 27/32 | -| EfficientNet-B0, ImageNet, 224x224 | 1,3,224,224 | standalone XTS3 | 190 int8 | 5.65 MB | 2352 KB | 2498.3 ms (2498.3) | 0.40 | bit-exact 2/2, ANSI too | top-1 19/32 (float 23), agrees with float 23/32 | -| LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224 | 1,3,224,224 | ExecuTorch .pte | 146 delegated, 0 CPU | 3.43 MB | 1045 KB | 3580.4 ms (3579.1); delegate 3580.5, CPU 0.0 | 0.28 | bit-exact | 6.1 dB vs float | -| SSDLite320 MobileNetV3 detection heads, COCO, 320x320 | 1,3,320,320 | ExecuTorch .pte | 243 delegated, 0 CPU | 3.72 MB | 2000 KB | 3830.3 ms (3830.0); delegate 3830.4, CPU 0.0 | 0.26 | bit-exact | 7.4 dB vs float | -| Small vision transformer: LayerNorm and attention on the CPU fallback | 1,3,64,64 | ExecuTorch .pte | 11 delegated, 30 CPU | 0.04 MB | 38 KB | 85.0 ms (84.9); delegate 22.1, CPU 62.8 | 11.77 | 39.3 dB vs host | top-1 agrees with float 31/32 | - -Latency is the model invocation only (no input preprocessing), timed over repeated runs after a bit-exact (or SQNR) check against the host reference. - -Host accuracy is the int8 model on the host (the board's output is bit-identical) after post-training quantization with 64 calibration images and no QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with random weights report top-1 agreement or SQNR against the float model instead. diff --git a/examples/espressif/espnn_bundle.py b/examples/espressif/espnn_bundle.py deleted file mode 100644 index 1eee34d3537..00000000000 --- a/examples/espressif/espnn_bundle.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -On-device test bundles for examples/espressif/espnn_runner. - -Bundle (little-endian), magic "XTB1" for XTS3 int8 graphs or "XTP1" for .pte programs: - magic u32 version=1 u32 count - count x { char name[32]; u32 blob_offset; u32 blob_size; u32 vectors_offset; u32 vectors_size } - payloads aligned to 128 bytes - -XTV2 vectors (XTS3 graphs; inputs and outputs in device layout, bit-exact): - "XTV2" u32 version=2 u32 samples u32 inputs u32 outputs - u32 input_bytes[inputs] u32 output_bytes[outputs] - samples x { int8 inputs..., int8 expected outputs... } - -XTE1 vectors (.pte programs; ExecuTorch tensors in logical layout): - "XTE1" u32 version=1 u32 samples u32 inputs u32 outputs f32 min_sqnr_db u32 flags - {u32 scalar_type, u32 numel} x (inputs + outputs) - samples x { inputs..., expected outputs... } - flags bit 0: outputs must be bit-identical; bit 1: outputs are stored as CRC32 only. -""" - -from __future__ import annotations - -import math -import struct -import zlib -from pathlib import Path - -import numpy as np -import torch - -from executorch.backends.espressif.reference import esp_nn_reference as reference - -NAME_BYTES = 32 -ALIGNMENT = 128 -FLAG_EXACT = 1 -FLAG_CRC = 2 -SCALAR_TYPES = {torch.float32: 6, torch.int8: 1, torch.int64: 4, torch.int32: 3} - - -def sqnr_db(reference_values: torch.Tensor, actual: torch.Tensor) -> float: - reference_values = reference_values.to(torch.float64) - noise = (reference_values - actual.to(torch.float64)).pow(2).sum().item() - if noise == 0: - return math.inf - return 10 * math.log10(reference_values.pow(2).sum().item() / noise) - - -def xtv2_vectors(graph, samples: list[list[np.ndarray]]) -> bytes: - """Quantize float samples and record the bit-exact outputs of the ESP-NN reference.""" - interpreter = reference.ReferenceInterpreter(graph) - input_tensors = [graph.tensors[i] for i in interpreter.inputs] - output_tensors = [graph.tensors[i] for i in interpreter.outputs] - input_bytes = [int(np.prod(reference.device_shape(t))) for t in input_tensors] - output_bytes = [int(np.prod(reference.device_shape(t))) for t in output_tensors] - body = bytearray() - for sample in samples: - device_inputs = [ - reference.to_device_layout(t, reference.quantize(t, np.asarray(value))) - for t, value in zip(input_tensors, sample) - ] - outputs = interpreter.run_outputs(device_inputs) - for value in device_inputs + outputs: - body += value.astype(np.int8).tobytes() - header = struct.pack( - "<4sIIII", b"XTV2", 2, len(samples), len(input_tensors), len(output_tensors) - ) - sizes = struct.pack( - f"<{len(input_bytes) + len(output_bytes)}I", *input_bytes, *output_bytes - ) - return header + sizes + bytes(body) - - -def xte1_vectors(rows, exact: bool, min_sqnr_db: float, crc: bool = False) -> bytes: - """rows: list of (inputs, outputs), each a list of tensors.""" - first_inputs, first_outputs = rows[0] - descriptors = [ - (SCALAR_TYPES[t.dtype], t.numel()) - for t in list(first_inputs) + list(first_outputs) - ] - flags = (FLAG_EXACT if exact else 0) | (FLAG_CRC if exact and crc else 0) - header = struct.pack( - "<4sIIIIfI", - b"XTE1", - 1, - len(rows), - len(first_inputs), - len(first_outputs), - float(min_sqnr_db), - flags, - ) - table = b"".join(struct.pack(" int: - """entries: list of (name, blob_bytes, vectors_bytes). Returns the bundle size in bytes.""" - if magic not in (b"XTB1", b"XTP1"): - raise ValueError(f"Unknown bundle magic {magic!r}") - payload = bytearray(b"\0" * (12 + len(entries) * (NAME_BYTES + 16))) - records = [] - for name, blob, vectors in entries: - payload.extend(b"\0" * ((-len(payload)) % ALIGNMENT)) - blob_offset = len(payload) - payload += blob - payload.extend(b"\0" * ((-len(payload)) % ALIGNMENT)) - vectors_offset = len(payload) - payload += vectors - records.append((name, blob_offset, len(blob), vectors_offset, len(vectors))) - header = struct.pack("<4sII", magic, 1, len(records)) - table = b"".join( - struct.pack(f"<{NAME_BYTES}sIIII", name.encode()[: NAME_BYTES - 1], *rest) - for name, *rest in records - ) - payload[: len(header) + len(table)] = header + table - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(bytes(payload)) - return len(payload) - - -def read_bundle(path: Path) -> tuple[bytes, list[tuple[str, bytes, bytes]]]: - data = path.read_bytes() - magic, version, count = struct.unpack_from("<4sII", data, 0) - entries = [] - for index in range(count): - name, blob_offset, blob_size, vectors_offset, vectors_size = struct.unpack_from( - f"<{NAME_BYTES}sIIII", data, 12 + index * (NAME_BYTES + 16) - ) - entries.append( - ( - name.rstrip(b"\0").decode(), - data[blob_offset : blob_offset + blob_size], - data[vectors_offset : vectors_offset + vectors_size], - ) - ) - return magic, entries - - -__all__ = ["read_bundle", "sqnr_db", "write_bundle", "xte1_vectors", "xtv2_vectors"] diff --git a/examples/espressif/espnn_runner/CMakeLists.txt b/examples/espressif/espnn_runner/CMakeLists.txt deleted file mode 100644 index 90ffb639948..00000000000 --- a/examples/espressif/espnn_runner/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -cmake_minimum_required(VERSION 3.16) - -get_filename_component( - EXECUTORCH_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE -) -set(EXTRA_COMPONENT_DIRS "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime") - -include($ENV{IDF_PATH}/tools/cmake/project.cmake) -project(espnn_runner) diff --git a/examples/espressif/espnn_runner/README.md b/examples/espressif/espnn_runner/README.md deleted file mode 100644 index 5f88a53d905..00000000000 --- a/examples/espressif/espnn_runner/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# ESP-NN runner - -ESP-IDF application that runs a test bundle on an Espressif board, checks every -model against the host (bit-exact for int8 graphs, bit-exact or SQNR for -ExecuTorch programs with CPU operators) and times it. - -Bundles are created by `../aot_espnn_compiler.py` (one model) or -`../scripts/make_test_bundles.py` (the backend's operator test models): - -| Bundle | Contents | Needs | -|---|---|---| -| `.xtb` (XTB1) | XTS3 int8 graphs run by the standalone `espnn_runtime` interpreter | nothing else | -| `.xtp` (XTP1) | ExecuTorch `.pte` programs, ESP-NN delegate plus portable CPU kernels | ExecuTorch cross-compiled for the chip, `CONFIG_ESPNN_EXECUTORCH_DELEGATE=y` | - -## Build and run - -From an activated ESP-IDF v5.1+ shell: - -```bash -cd examples/espressif/espnn_runner -python ../scripts/make_test_bundles.py # bundles/op_suite.xtb, bundles/pte_suite.xtp - -# Standalone int8 graphs (any ESP32 chip) -idf.py set-target esp32s3 # or esp32 -idf.py -DSDKCONFIG_DEFAULTS="sdkconfig.defaults" menuconfig # ESP-NN runner -> bundle path -idf.py -p /dev/ttyACM0 flash monitor - -# ExecuTorch programs (needs ../scripts/build_executorch_esp.sh esp32s3 first) -idf.py -B build-et -DSDKCONFIG=build-et/sdkconfig \ - -DSDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.executorch" \ - -DESPNN_EXECUTORCH_BUILD_DIR=$EXECUTORCH_ROOT/cmake-out-esp-esp32s3 build flash monitor -``` - -`../scripts/run_bundle.py BUNDLE --port PORT` does the same (target detection, -build, flash, capture) and prints a summary. - -Output, one line per model: - -``` -CHIP target=esp32s3 cores=2 revision=2 cpu_mhz=240 flash_mb=16 psram_kb=8192 ... -SUITE CASE 1/11 name=conv-act-pool-linear ops=9 ... exact=3 ... ansi=1 us=2444 min_us=... status=PASS -ET CASE 5/7 name=tiny-transformer ... delegate_us=2288 cpu_us=6040 sqnr_db=41.6 ... status=PASS -ESPNN RUNNER: PASS -``` - -`sdkconfig.defaults.esp32s3` assumes 16 MB flash and octal PSRAM (for example -ESP32-S3-DevKitC-1 N16R8). For other modules change the flash size, partition -table and PSRAM mode. Boards without PSRAM still boot (`CONFIG_SPIRAM_IGNORE_NOTFOUND`) -and run models whose activations fit in internal RAM. diff --git a/examples/espressif/espnn_runner/main/CMakeLists.txt b/examples/espressif/espnn_runner/main/CMakeLists.txt deleted file mode 100644 index 025ffd7b69f..00000000000 --- a/examples/espressif/espnn_runner/main/CMakeLists.txt +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -set(runner_sources "main.cpp" "int8_suite.cpp" "bundle_data.S") -if(CONFIG_ESPNN_EXECUTORCH_DELEGATE) - list(APPEND runner_sources "pte_suite.cpp" - "${CMAKE_CURRENT_LIST_DIR}/../../executor_runner/esp_pal.cpp" - ) -endif() - -idf_component_register( - SRCS - ${runner_sources} - INCLUDE_DIRS - "." - REQUIRES - espnn_runtime - esp_timer - heap - esp_rom - spi_flash -) - -idf_build_get_property(project_dir PROJECT_DIR) -if(IS_ABSOLUTE "${CONFIG_ESPNN_RUNNER_BUNDLE}") - set(bundle "${CONFIG_ESPNN_RUNNER_BUNDLE}") -else() - set(bundle "${project_dir}/${CONFIG_ESPNN_RUNNER_BUNDLE}") -endif() -if(NOT EXISTS "${bundle}") - message( - FATAL_ERROR - "Test bundle ${bundle} not found. Create one with " - "examples/espressif/aot_espnn_compiler.py or examples/espressif/scripts/make_*.py " - "and set CONFIG_ESPNN_RUNNER_BUNDLE." - ) -endif() -set_source_files_properties( - "bundle_data.S" - PROPERTIES COMPILE_DEFINITIONS "ESPNN_BUNDLE_FILE=\"${bundle}\"" - OBJECT_DEPENDS "${bundle}" -) diff --git a/examples/espressif/espnn_runner/main/Kconfig.projbuild b/examples/espressif/espnn_runner/main/Kconfig.projbuild deleted file mode 100644 index 733f6646986..00000000000 --- a/examples/espressif/espnn_runner/main/Kconfig.projbuild +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -menu "ESP-NN runner" - - config ESPNN_RUNNER_BUNDLE - string "Test bundle to embed (XTB1 int8 graphs or XTP1 .pte programs)" - default "bundles/op_suite.xtb" - help - Absolute, or relative to this project directory. - - config ESPNN_RUNNER_BENCH_ITERATIONS - int "Timed iterations per model after the correctness check" - range 0 1000 - default 10 - - config ESPNN_RUNNER_VERIFY_ANSI - bool "Also run each int8 graph once with the ESP-NN ANSI reference kernels" - default y - - config ESPNN_RUNNER_DUAL_CORE - bool "Split ESP-NN kernels across both cores (int8 graphs)" - default n - depends on !FREERTOS_UNICORE - - config ESPNN_RUNNER_CONSTANTS_TO_PSRAM - bool "Copy int8 graph weights from flash into PSRAM when they fit" - default y - depends on SPIRAM - - config ESPNN_RUNNER_METHOD_POOL_KB - int "ExecuTorch method allocator pool (KB)" - default 1024 - depends on ESPNN_EXECUTORCH_DELEGATE - - config ESPNN_RUNNER_TEMP_POOL_KB - int "ExecuTorch temporary allocator pool (KB)" - default 512 - depends on ESPNN_EXECUTORCH_DELEGATE - - config ESPNN_RUNNER_STACK_KB - int "Runner task stack (KB)" - default 48 if ESPNN_EXECUTORCH_DELEGATE - default 16 - -endmenu diff --git a/examples/espressif/espnn_runner/main/bundle_data.S b/examples/espressif/espnn_runner/main/bundle_data.S deleted file mode 100644 index e16fce020f5..00000000000 --- a/examples/espressif/espnn_runner/main/bundle_data.S +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - - .section .rodata.espnn_bundle, "a" - .balign 128 - .global espnn_bundle_start -espnn_bundle_start: - .incbin ESPNN_BUNDLE_FILE - .global espnn_bundle_end -espnn_bundle_end: - .byte 0 - - .section .note.GNU-stack, "", @progbits diff --git a/examples/espressif/espnn_runner/main/int8_suite.cpp b/examples/espressif/espnn_runner/main/int8_suite.cpp deleted file mode 100644 index a40ab037b4e..00000000000 --- a/examples/espressif/espnn_runner/main/int8_suite.cpp +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -#include -#include -#include - -#include -#include -#include - -#include - -namespace executorch { -namespace examples { -namespace espressif { - -using ::executorch::backends::espressif::Int8Interpreter; -using ::executorch::backends::espressif::Int8RuntimeOptions; -using ::executorch::backends::espressif::Int8TensorInfo; -using ::executorch::backends::espressif::KernelMode; -using ::executorch::backends::espressif::MAX_GRAPH_IO; -using ::executorch::backends::espressif::read_le_u32; - -namespace { - -constexpr size_t BUNDLE_HEADER_SIZE = 12; -constexpr size_t BUNDLE_NAME_BYTES = 32; -constexpr size_t BUNDLE_RECORD_SIZE = BUNDLE_NAME_BYTES + 16; -constexpr size_t XTV2_HEADER_SIZE = 20; - -bool in_bounds(size_t offset, size_t length, size_t total) { - return offset <= total && length <= total - offset; -} - -struct VectorsV2 { - uint32_t samples; - uint32_t inputs; - uint32_t outputs; - uint32_t input_bytes[MAX_GRAPH_IO]; - uint32_t output_bytes[MAX_GRAPH_IO]; - size_t stride; - const uint8_t* data; -}; - -bool parse_vectors_v2(const uint8_t* data, size_t size, VectorsV2& v) { - if (size < XTV2_HEADER_SIZE || std::memcmp(data, "XTV2", 4) != 0 || - read_le_u32(data + 4) != 2) { - return false; - } - v.samples = read_le_u32(data + 8); - v.inputs = read_le_u32(data + 12); - v.outputs = read_le_u32(data + 16); - if (v.samples == 0 || v.inputs == 0 || v.outputs == 0 || - v.inputs > MAX_GRAPH_IO || v.outputs > MAX_GRAPH_IO) - return false; - const size_t table = XTV2_HEADER_SIZE + 4U * (v.inputs + v.outputs); - if (size < table) - return false; - v.stride = 0; - for (uint32_t i = 0; i < v.inputs; ++i) { - v.input_bytes[i] = read_le_u32(data + XTV2_HEADER_SIZE + 4 * i); - v.stride += v.input_bytes[i]; - } - for (uint32_t i = 0; i < v.outputs; ++i) { - v.output_bytes[i] = - read_le_u32(data + XTV2_HEADER_SIZE + 4 * (v.inputs + i)); - v.stride += v.output_bytes[i]; - } - v.data = data + table; - return v.stride != 0 && (size - table) / v.stride >= v.samples && - table + v.stride * v.samples == size; -} - -struct CaseResult { - uint32_t exact_samples; - uint32_t mismatched; - int32_t worst; - bool ansi_exact; - int64_t total_us; - int64_t bench_min_us; - int64_t bench_max_us; - int64_t bench_total_us; - uint32_t bench_iterations; -}; - -uint32_t compare(const int8_t* a, const int8_t* b, size_t n, int32_t& worst) { - uint32_t mismatches = 0; - for (size_t i = 0; i < n; ++i) { - int32_t d = static_cast(a[i]) - b[i]; - if (d < 0) - d = -d; - if (d != 0) - ++mismatches; - if (d > worst) - worst = d; - } - return mismatches; -} - -bool run_sample( - Int8Interpreter& interpreter, - const VectorsV2& v, - uint32_t index, - uint32_t& mismatched, - int32_t& worst) { - const uint8_t* sample = v.data + v.stride * index; - for (uint32_t i = 0; i < v.inputs; ++i) { - std::memcpy(interpreter.input_data(i), sample, v.input_bytes[i]); - sample += v.input_bytes[i]; - } - if (!interpreter.invoke()) - return false; - mismatched = 0; - for (uint32_t o = 0; o < v.outputs; ++o) { - mismatched += compare( - interpreter.output_data(o), - reinterpret_cast(sample), - v.output_bytes[o], - worst); - sample += v.output_bytes[o]; - } - return true; -} - -bool run_case( - const char* name, - const uint8_t* blob, - size_t blob_size, - const uint8_t* vectors, - size_t vectors_size, - uint32_t index, - uint32_t count) { - VectorsV2 v{}; - if (!parse_vectors_v2(vectors, vectors_size, v)) { - std::printf( - "SUITE CASE %u/%u name=%s status=FAIL reason=vectors\n", - static_cast(index), - static_cast(count), - name); - return false; - } - Int8Interpreter* interpreter = new (std::nothrow) Int8Interpreter(); - Int8RuntimeOptions options{}; -#if CONFIG_ESPNN_RUNNER_DUAL_CORE - options.dual_core = true; -#endif -#if CONFIG_ESPNN_RUNNER_CONSTANTS_TO_PSRAM - options.copy_constants_to_psram = true; -#endif - if (interpreter == nullptr || !interpreter->load(blob, blob_size, options)) { - std::printf( - "SUITE CASE %u/%u name=%s status=FAIL reason=load\n", - static_cast(index), - static_cast(count), - name); - delete interpreter; - return false; - } - bool shapes_ok = interpreter->input_count() == v.inputs && - interpreter->output_count() == v.outputs; - for (uint32_t i = 0; shapes_ok && i < v.inputs; ++i) { - Int8TensorInfo info{}; - shapes_ok = - interpreter->input_info(i, info) && info.bytes == v.input_bytes[i]; - } - for (uint32_t o = 0; shapes_ok && o < v.outputs; ++o) { - Int8TensorInfo info{}; - shapes_ok = - interpreter->output_info(o, info) && info.bytes == v.output_bytes[o]; - } - if (!shapes_ok) { - std::printf( - "SUITE CASE %u/%u name=%s status=FAIL reason=io-shapes\n", - static_cast(index), - static_cast(count), - name); - delete interpreter; - return false; - } - CaseResult result{}; - bool ok = true; - for (uint32_t s = 0; ok && s < v.samples; ++s) { - uint32_t mismatched = 0; - ok = run_sample(*interpreter, v, s, mismatched, result.worst); - result.mismatched += mismatched; - if (ok && mismatched == 0) - ++result.exact_samples; - result.total_us += interpreter->last_invoke_us(); - } - result.ansi_exact = true; -#if CONFIG_ESPNN_RUNNER_VERIFY_ANSI - if (ok) { - uint32_t mismatched = 0; - int32_t worst = 0; - interpreter->set_kernel_mode(KernelMode::REFERENCE); - ok = run_sample(*interpreter, v, 0, mismatched, worst); - interpreter->set_kernel_mode(KernelMode::OPTIMIZED); - result.ansi_exact = ok && mismatched == 0; - } -#endif - result.bench_min_us = INT64_MAX; - for (uint32_t i = 0; ok && i < CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS; ++i) { - ok = interpreter->invoke(); - const int64_t elapsed = interpreter->last_invoke_us(); - result.bench_total_us += elapsed; - if (elapsed < result.bench_min_us) - result.bench_min_us = elapsed; - if (elapsed > result.bench_max_us) - result.bench_max_us = elapsed; - ++result.bench_iterations; - } - if (result.bench_iterations == 0) - result.bench_min_us = 0; - const bool pass = - ok && result.exact_samples == v.samples && result.ansi_exact; - std::printf( - "SUITE CASE %u/%u name=%s ops=%u inputs=%u outputs=%u samples=%u exact=%u " - "mismatched=%u worst=%ld ansi=%d us=%lld min_us=%lld max_us=%lld iterations=%u " - "arena=%u arena_internal=%d constants_in_ram=%d blob=%u status=%s\n", - static_cast(index), - static_cast(count), - name, - static_cast(interpreter->graph().header.num_operations), - static_cast(v.inputs), - static_cast(v.outputs), - static_cast(v.samples), - static_cast(result.exact_samples), - static_cast(result.mismatched), - static_cast(result.worst), - result.ansi_exact ? 1 : 0, - static_cast( - result.bench_iterations - ? result.bench_total_us / result.bench_iterations - : result.total_us / v.samples), - static_cast(result.bench_min_us), - static_cast(result.bench_max_us), - static_cast(result.bench_iterations), - static_cast(interpreter->arena_bytes()), - interpreter->arena_in_internal_ram() ? 1 : 0, - interpreter->constants_copied() ? 1 : 0, - static_cast(blob_size), - pass ? "PASS" : "FAIL"); - if (!pass) - interpreter->print_profile(); - delete interpreter; - return pass; -} - -} // namespace - -bool run_int8_suite(const uint8_t* data, size_t size) { - std::printf("\n========================================\n"); - std::printf(" ESP-NN INT8 OPERATOR SUITE\n"); - std::printf("========================================\n"); - if (!is_int8_bundle(data, size) || read_le_u32(data + 4) != 1) { - std::printf("SUITE RESULT: FAIL reason=bundle\n"); - return false; - } - const uint32_t count = read_le_u32(data + 8); - if (count == 0 || - !in_bounds( - BUNDLE_HEADER_SIZE, - static_cast(count) * BUNDLE_RECORD_SIZE, - size)) { - std::printf("SUITE RESULT: FAIL reason=bundle-table\n"); - return false; - } - uint32_t passed = 0; - const int64_t started = esp_timer_get_time(); - for (uint32_t i = 0; i < count; ++i) { - const uint8_t* record = - data + BUNDLE_HEADER_SIZE + static_cast(i) * BUNDLE_RECORD_SIZE; - char name[BUNDLE_NAME_BYTES + 1]{}; - std::memcpy(name, record, BUNDLE_NAME_BYTES); - const uint32_t blob_offset = read_le_u32(record + BUNDLE_NAME_BYTES); - const uint32_t blob_size = read_le_u32(record + BUNDLE_NAME_BYTES + 4); - const uint32_t vectors_offset = read_le_u32(record + BUNDLE_NAME_BYTES + 8); - const uint32_t vectors_size = read_le_u32(record + BUNDLE_NAME_BYTES + 12); - if (!in_bounds(blob_offset, blob_size, size) || - !in_bounds(vectors_offset, vectors_size, size)) { - std::printf( - "SUITE CASE %u/%u name=%s status=FAIL reason=bounds\n", - static_cast(i + 1), - static_cast(count), - name); - continue; - } - if (run_case( - name, - data + blob_offset, - blob_size, - data + vectors_offset, - vectors_size, - i + 1, - count)) - ++passed; - } - std::printf( - "Internal RAM free %u, PSRAM free %u, suite time %lld ms\n", - static_cast( - heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), - static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM)), - static_cast((esp_timer_get_time() - started) / 1000)); - std::printf( - "SUITE RESULT: %s cases=%u passed=%u\n", - passed == count ? "PASS" : "FAIL", - static_cast(count), - static_cast(passed)); - return passed == count; -} - -} // namespace espressif -} // namespace examples -} // namespace executorch diff --git a/examples/espressif/espnn_runner/main/main.cpp b/examples/espressif/espnn_runner/main/main.cpp deleted file mode 100644 index 12bdd031f74..00000000000 --- a/examples/espressif/espnn_runner/main/main.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -extern "C" { -extern const uint8_t espnn_bundle_start[]; -extern const uint8_t espnn_bundle_end[]; -} - -namespace { - -void print_chip() { - esp_chip_info_t chip{}; - esp_chip_info(&chip); - uint32_t flash_bytes = 0; - esp_flash_get_size(nullptr, &flash_bytes); - std::printf( - "CHIP target=%s cores=%d revision=%d cpu_mhz=%d flash_mb=%u psram_kb=%u " - "internal_free_kb=%u idf=%s\n", - CONFIG_IDF_TARGET, - chip.cores, - chip.revision, - CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ, - static_cast(flash_bytes / (1024 * 1024)), - static_cast(heap_caps_get_total_size(MALLOC_CAP_SPIRAM) / 1024), - static_cast( - heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) / - 1024), - esp_get_idf_version()); -} - -void runner_task(void*) { - print_chip(); - const size_t size = reinterpret_cast(espnn_bundle_end) - - reinterpret_cast(espnn_bundle_start); - bool pass = false; - if (executorch::examples::espressif::is_int8_bundle( - espnn_bundle_start, size)) { - pass = executorch::examples::espressif::run_int8_suite( - espnn_bundle_start, size); - } else if (executorch::examples::espressif::is_pte_bundle( - espnn_bundle_start, size)) { -#if CONFIG_ESPNN_EXECUTORCH_DELEGATE - pass = executorch::examples::espressif::run_pte_suite( - espnn_bundle_start, size); -#else - std::printf( - "ERROR: this bundle holds .pte programs; enable CONFIG_ESPNN_EXECUTORCH_DELEGATE\n"); -#endif - } else { - std::printf( - "ERROR: unrecognised bundle; expected XTB1 (int8 graphs) or XTP1 (.pte programs)\n"); - } - std::printf("ESPNN RUNNER: %s\n", pass ? "PASS" : "FAIL"); - while (true) - vTaskDelay(pdMS_TO_TICKS(10000)); -} - -} // namespace - -extern "C" void app_main() { - xTaskCreatePinnedToCore( - runner_task, - "espnn_runner", - CONFIG_ESPNN_RUNNER_STACK_KB * 1024, - nullptr, - 5, - nullptr, - 0); -} diff --git a/examples/espressif/espnn_runner/main/pte_suite.cpp b/examples/espressif/espnn_runner/main/pte_suite.cpp deleted file mode 100644 index cf010387464..00000000000 --- a/examples/espressif/espnn_runner/main/pte_suite.cpp +++ /dev/null @@ -1,494 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -using executorch::aten::ScalarType; -using executorch::aten::Tensor; -using executorch::extension::BufferDataLoader; -using executorch::runtime::Error; -using executorch::runtime::EValue; -using executorch::runtime::HierarchicalAllocator; -using executorch::runtime::MemoryAllocator; -using executorch::runtime::MemoryManager; -using executorch::runtime::Method; -using executorch::runtime::MethodMeta; -using executorch::runtime::Program; -using executorch::runtime::Result; -using executorch::runtime::Span; - -namespace executorch { -namespace examples { -namespace espressif { - -using ::executorch::backends::espressif::delegate_stats; -using ::executorch::backends::espressif::DelegateStats; -using ::executorch::backends::espressif::read_le_u32; -using ::executorch::backends::espressif::register_espnn_backend; -using ::executorch::backends::espressif::reset_delegate_stats; - -namespace { - -constexpr size_t BUNDLE_HEADER_SIZE = 12; -constexpr size_t BUNDLE_NAME_BYTES = 32; -constexpr size_t BUNDLE_RECORD_SIZE = BUNDLE_NAME_BYTES + 16; -constexpr size_t XTE1_HEADER_SIZE = 28; -constexpr uint32_t XTE1_FLAG_EXACT = 1; -constexpr uint32_t XTE1_FLAG_CRC = 2; -constexpr size_t MAX_IO = 16; -constexpr size_t MAX_PLANNED_BUFFERS = 8; - -bool in_bounds(size_t offset, size_t length, size_t total) { - return offset <= total && length <= total - offset; -} - -size_t element_size(uint32_t dtype) { - if (dtype == static_cast(ScalarType::Float)) - return 4; - if (dtype == static_cast(ScalarType::Char)) - return 1; - if (dtype == static_cast(ScalarType::Long)) - return 8; - if (dtype == static_cast(ScalarType::Int)) - return 4; - return 0; -} - -struct Vectors { - uint32_t samples; - uint32_t inputs; - uint32_t outputs; - float min_sqnr_db; - uint32_t flags; - uint32_t dtype[2 * MAX_IO]; - uint32_t numel[2 * MAX_IO]; - size_t stride; - const uint8_t* data; -}; - -bool parse_vectors(const uint8_t* data, size_t size, Vectors& v) { - if (size < XTE1_HEADER_SIZE || std::memcmp(data, "XTE1", 4) != 0 || - read_le_u32(data + 4) != 1) { - return false; - } - v.samples = read_le_u32(data + 8); - v.inputs = read_le_u32(data + 12); - v.outputs = read_le_u32(data + 16); - const uint32_t sqnr_bits = read_le_u32(data + 20); - std::memcpy(&v.min_sqnr_db, &sqnr_bits, sizeof(float)); - v.flags = read_le_u32(data + 24); - if (v.samples == 0 || v.inputs == 0 || v.outputs == 0 || v.inputs > MAX_IO || - v.outputs > MAX_IO) { - return false; - } - const size_t table = XTE1_HEADER_SIZE + 8U * (v.inputs + v.outputs); - if (size < table) - return false; - v.stride = 0; - for (uint32_t i = 0; i < v.inputs + v.outputs; ++i) { - v.dtype[i] = read_le_u32(data + XTE1_HEADER_SIZE + 8 * i); - v.numel[i] = read_le_u32(data + XTE1_HEADER_SIZE + 8 * i + 4); - const size_t bytes = element_size(v.dtype[i]) * v.numel[i]; - if (bytes == 0) - return false; - v.stride += (i >= v.inputs && (v.flags & XTE1_FLAG_CRC) != 0) ? 4U : bytes; - } - v.data = data + table; - return (size - table) / v.stride >= v.samples && - table + v.stride * v.samples == size; -} - -struct Pools { - uint8_t* method_pool = nullptr; - uint8_t* temp_pool = nullptr; - uint8_t* planned[MAX_PLANNED_BUFFERS]{}; - size_t planned_count = 0; - size_t planned_bytes = 0; - - ~Pools() { - if (method_pool != nullptr) - heap_caps_free(method_pool); - if (temp_pool != nullptr) - heap_caps_free(temp_pool); - for (size_t i = 0; i < planned_count; ++i) { - if (planned[i] != nullptr) - heap_caps_free(planned[i]); - } - } -}; - -void* psram(size_t bytes) { - void* pointer = - heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); - if (pointer == nullptr) - pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_8BIT); - return pointer; -} - -struct OutputScore { - double signal; - double noise; - float max_abs; - uint32_t mismatched; -}; - -void score_output( - const Tensor& actual, - uint32_t dtype, - const uint8_t* expected, - size_t count, - OutputScore& score) { - if (dtype == static_cast(ScalarType::Float)) { - const float* got = actual.const_data_ptr(); - for (size_t i = 0; i < count; ++i) { - float want = 0.0f; - std::memcpy(&want, expected + 4 * i, sizeof(float)); - const double difference = static_cast(got[i]) - want; - score.signal += static_cast(want) * want; - score.noise += difference * difference; - const float magnitude = static_cast(std::fabs(difference)); - if (magnitude > score.max_abs) - score.max_abs = magnitude; - if (got[i] != want) - ++score.mismatched; - } - return; - } - const uint8_t* got = static_cast(actual.const_data_ptr()); - const size_t bytes = count * element_size(dtype); - for (size_t i = 0; i < bytes; ++i) { - if (got[i] != expected[i]) - ++score.mismatched; - } -} - -bool run_case( - const char* name, - const uint8_t* pte, - size_t pte_size, - const uint8_t* vectors, - size_t vectors_size, - uint32_t index, - uint32_t count) { - Vectors v{}; - if (!parse_vectors(vectors, vectors_size, v)) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=vectors\n", - static_cast(index), - static_cast(count), - name); - return false; - } - reset_delegate_stats(); - const int64_t load_started = esp_timer_get_time(); - BufferDataLoader loader(pte, pte_size); - Result program = Program::load(&loader); - if (!program.ok()) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=program-load error=0x%x\n", - static_cast(index), - static_cast(count), - name, - static_cast(program.error())); - return false; - } - Result meta = program->method_meta("forward"); - if (!meta.ok() || meta->num_memory_planned_buffers() > MAX_PLANNED_BUFFERS) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=method-meta\n", - static_cast(index), - static_cast(count), - name); - return false; - } - Pools pools; - const size_t method_bytes = - static_cast(CONFIG_ESPNN_RUNNER_METHOD_POOL_KB) * 1024U; - const size_t temp_bytes = - static_cast(CONFIG_ESPNN_RUNNER_TEMP_POOL_KB) * 1024U; - pools.method_pool = static_cast(psram(method_bytes)); - pools.temp_pool = static_cast(psram(temp_bytes)); - Span spans[MAX_PLANNED_BUFFERS]; - pools.planned_count = meta->num_memory_planned_buffers(); - bool allocated = pools.method_pool != nullptr && pools.temp_pool != nullptr; - for (size_t i = 0; allocated && i < pools.planned_count; ++i) { - const size_t bytes = - static_cast(meta->memory_planned_buffer_size(i).get()); - pools.planned[i] = static_cast(psram(bytes == 0 ? 16 : bytes)); - allocated = pools.planned[i] != nullptr; - spans[i] = Span(pools.planned[i], bytes); - pools.planned_bytes += bytes; - } - if (!allocated) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=memory\n", - static_cast(index), - static_cast(count), - name); - return false; - } - MemoryAllocator method_allocator( - static_cast(method_bytes), pools.method_pool); - MemoryAllocator temp_allocator( - static_cast(temp_bytes), pools.temp_pool); - HierarchicalAllocator planned( - Span>(spans, pools.planned_count)); - MemoryManager memory(&method_allocator, &planned, &temp_allocator); - bool pass = false; - { - Result method = program->load_method("forward", &memory); - if (!method.ok()) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=method-load error=0x%x\n", - static_cast(index), - static_cast(count), - name, - static_cast(method.error())); - return false; - } - const int64_t load_us = esp_timer_get_time() - load_started; - if (method->inputs_size() != v.inputs || - method->outputs_size() != v.outputs) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=io-count\n", - static_cast(index), - static_cast(count), - name); - return false; - } - EValue inputs[MAX_IO]; - OutputScore scores[MAX_IO]{}; - int64_t total_us = 0; - int64_t fastest = INT64_MAX; - bool ok = true; - for (uint32_t s = 0; ok && s < v.samples; ++s) { - const uint8_t* sample = v.data + v.stride * s; - ok = method->get_inputs(inputs, v.inputs) == Error::Ok; - for (uint32_t i = 0; ok && i < v.inputs; ++i) { - Tensor tensor = inputs[i].toTensor(); - const size_t bytes = element_size(v.dtype[i]) * v.numel[i]; - ok = static_cast(tensor.scalar_type()) == v.dtype[i] && - tensor.nbytes() == bytes; - if (ok) - std::memcpy(tensor.mutable_data_ptr(), sample, bytes); - sample += bytes; - } - if (!ok) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=input-shape\n", - static_cast(index), - static_cast(count), - name); - return false; - } - const int64_t started = esp_timer_get_time(); - const Error status = method->execute(); - const int64_t elapsed = esp_timer_get_time() - started; - if (status != Error::Ok) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=execute error=0x%x\n", - static_cast(index), - static_cast(count), - name, - static_cast(status)); - return false; - } - total_us += elapsed; - if (elapsed < fastest) - fastest = elapsed; - for (uint32_t o = 0; o < v.outputs; ++o) { - const uint32_t slot = v.inputs + o; - const Tensor tensor = method->get_output(o).toTensor(); - const size_t bytes = element_size(v.dtype[slot]) * v.numel[slot]; - if (static_cast(tensor.scalar_type()) != v.dtype[slot] || - tensor.nbytes() != bytes) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=output-shape\n", - static_cast(index), - static_cast(count), - name); - return false; - } - if ((v.flags & XTE1_FLAG_CRC) != 0) { - const uint32_t crc = esp_rom_crc32_le( - 0, static_cast(tensor.const_data_ptr()), bytes); - if (crc != read_le_u32(sample)) - ++scores[o].mismatched; - sample += 4; - continue; - } - score_output(tensor, v.dtype[slot], sample, v.numel[slot], scores[o]); - sample += bytes; - } - } - int64_t bench_min = INT64_MAX; - int64_t bench_max = 0; - int64_t bench_total = 0; - uint32_t bench_iterations = 0; - for (uint32_t i = 0; ok && i < CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS; ++i) { - const int64_t started = esp_timer_get_time(); - ok = method->execute() == Error::Ok; - const int64_t elapsed = esp_timer_get_time() - started; - bench_total += elapsed; - if (elapsed < bench_min) - bench_min = elapsed; - if (elapsed > bench_max) - bench_max = elapsed; - ++bench_iterations; - } - if (bench_iterations == 0) - bench_min = fastest; - double worst_sqnr = INFINITY; - float max_abs = 0.0f; - uint32_t mismatched = 0; - for (uint32_t o = 0; o < v.outputs; ++o) { - mismatched += scores[o].mismatched; - if (scores[o].max_abs > max_abs) - max_abs = scores[o].max_abs; - if (scores[o].noise > 0.0 && scores[o].signal > 0.0) { - const double sqnr = - 10.0 * std::log10(scores[o].signal / scores[o].noise); - if (sqnr < worst_sqnr) - worst_sqnr = sqnr; - } else if (scores[o].noise > 0.0) { - worst_sqnr = -INFINITY; - } - } - const DelegateStats stats = delegate_stats(); - const uint32_t runs = v.samples + bench_iterations; - const int64_t mean_us = bench_iterations ? bench_total / bench_iterations - : total_us / v.samples; - const int64_t delegate_us = stats.calls ? stats.total_us / runs : 0; - pass = (v.flags & XTE1_FLAG_EXACT) != 0 ? mismatched == 0 - : worst_sqnr >= v.min_sqnr_db; - std::printf( - "ET CASE %u/%u name=%s inputs=%u outputs=%u samples=%u delegates=%u " - "delegate_calls=%u us=%lld min_us=%lld max_us=%lld iterations=%u delegate_us=%lld " - "cpu_us=%lld sqnr_db=%.1f " - "max_abs=%.6g mismatched=%u exact_required=%d min_sqnr=%.1f load_us=%lld " - "planned=%u arena=%u status=%s\n", - static_cast(index), - static_cast(count), - name, - static_cast(v.inputs), - static_cast(v.outputs), - static_cast(v.samples), - static_cast(stats.delegates), - static_cast(stats.calls / runs), - static_cast(mean_us), - static_cast(bench_min < fastest ? bench_min : fastest), - static_cast(bench_iterations ? bench_max : mean_us), - static_cast(bench_iterations), - static_cast(delegate_us), - static_cast(mean_us - delegate_us), - std::isinf(worst_sqnr) ? (worst_sqnr > 0 ? 999.0 : -999.0) : worst_sqnr, - static_cast(max_abs), - static_cast(mismatched), - (v.flags & XTE1_FLAG_EXACT) != 0 ? 1 : 0, - static_cast(v.min_sqnr_db), - static_cast(load_us), - static_cast(pools.planned_bytes), - static_cast(stats.arena_bytes), - pass ? "PASS" : "FAIL"); - } - return pass; -} - -} // namespace - -bool run_pte_suite(const uint8_t* data, size_t size) { - std::printf("\n========================================\n"); - std::printf( - " EXECUTORCH RUNTIME SUITE (ESP-NN delegate + portable CPU kernels)\n"); - std::printf("========================================\n"); - executorch::runtime::runtime_init(); - if (!register_espnn_backend()) { - std::printf("ET RESULT: FAIL reason=backend-registration\n"); - return false; - } - if (!is_pte_bundle(data, size) || read_le_u32(data + 4) != 1) { - std::printf("ET RESULT: FAIL reason=bundle\n"); - return false; - } - const uint32_t count = read_le_u32(data + 8); - if (count == 0 || - !in_bounds( - BUNDLE_HEADER_SIZE, - static_cast(count) * BUNDLE_RECORD_SIZE, - size)) { - std::printf("ET RESULT: FAIL reason=bundle-table\n"); - return false; - } - uint32_t passed = 0; - const int64_t started = esp_timer_get_time(); - for (uint32_t i = 0; i < count; ++i) { - const uint8_t* record = - data + BUNDLE_HEADER_SIZE + static_cast(i) * BUNDLE_RECORD_SIZE; - char name[BUNDLE_NAME_BYTES + 1]{}; - std::memcpy(name, record, BUNDLE_NAME_BYTES); - const uint32_t pte_offset = read_le_u32(record + BUNDLE_NAME_BYTES); - const uint32_t pte_size = read_le_u32(record + BUNDLE_NAME_BYTES + 4); - const uint32_t vectors_offset = read_le_u32(record + BUNDLE_NAME_BYTES + 8); - const uint32_t vectors_size = read_le_u32(record + BUNDLE_NAME_BYTES + 12); - if (!in_bounds(pte_offset, pte_size, size) || - !in_bounds(vectors_offset, vectors_size, size)) { - std::printf( - "ET CASE %u/%u name=%s status=FAIL reason=bounds\n", - static_cast(i + 1), - static_cast(count), - name); - continue; - } - if (run_case( - name, - data + pte_offset, - pte_size, - data + vectors_offset, - vectors_size, - i + 1, - count)) { - ++passed; - } - std::printf( - "ET MEMORY after=%s internal_free=%u psram_free=%u\n", - name, - static_cast( - heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), - static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM))); - } - std::printf( - "ET RESULT: %s cases=%u passed=%u time_ms=%lld\n", - passed == count ? "PASS" : "FAIL", - static_cast(count), - static_cast(passed), - static_cast((esp_timer_get_time() - started) / 1000)); - return passed == count; -} - -} // namespace espressif -} // namespace examples -} // namespace executorch diff --git a/examples/espressif/espnn_runner/main/runner.h b/examples/espressif/espnn_runner/main/runner.h deleted file mode 100644 index a881dcf3218..00000000000 --- a/examples/espressif/espnn_runner/main/runner.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2026 Mwaniki Njagi. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#pragma once - -#include -#include -#include - -namespace executorch { -namespace examples { -namespace espressif { - -/// Whether @p data is an XTB1 bundle of XTS3 int8 graphs. -inline bool is_int8_bundle(const uint8_t* data, size_t size) { - return data != nullptr && size >= 12 && std::memcmp(data, "XTB1", 4) == 0; -} - -/// Whether @p data is an XTP1 bundle of ExecuTorch programs. -inline bool is_pte_bundle(const uint8_t* data, size_t size) { - return data != nullptr && size >= 12 && std::memcmp(data, "XTP1", 4) == 0; -} - -/** - * Runs every graph in an XTB1 bundle, checks it against the bundled vectors - * and prints one SUITE CASE line per graph. - * - * @retval true Every case passed. - */ -bool run_int8_suite(const uint8_t* data, size_t size); -/** - * Loads every program in an XTP1 bundle with the ExecuTorch runtime, checks - * it against the bundled vectors and prints one ET CASE line per program. - * - * @retval true Every case passed. - */ -bool run_pte_suite(const uint8_t* data, size_t size); - -} // namespace espressif -} // namespace examples -} // namespace executorch diff --git a/examples/espressif/espnn_runner/partitions_16mb.csv b/examples/espressif/espnn_runner/partitions_16mb.csv deleted file mode 100644 index 195f4193a6a..00000000000 --- a/examples/espressif/espnn_runner/partitions_16mb.csv +++ /dev/null @@ -1,3 +0,0 @@ -# Name, Type, SubType, Offset, Size -nvs, data, nvs, 0x9000, 0x6000 -factory, app, factory, 0x10000, 15M diff --git a/examples/espressif/espnn_runner/partitions_4mb.csv b/examples/espressif/espnn_runner/partitions_4mb.csv deleted file mode 100644 index e1a3fef0195..00000000000 --- a/examples/espressif/espnn_runner/partitions_4mb.csv +++ /dev/null @@ -1,3 +0,0 @@ -# Name, Type, SubType, Offset, Size -nvs, data, nvs, 0x9000, 0x6000 -factory, app, factory, 0x10000, 0x3F0000 diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults b/examples/espressif/espnn_runner/sdkconfig.defaults deleted file mode 100644 index 84c7e8732ab..00000000000 --- a/examples/espressif/espnn_runner/sdkconfig.defaults +++ /dev/null @@ -1,11 +0,0 @@ -CONFIG_COMPILER_OPTIMIZATION_PERF=y -CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y -CONFIG_PARTITION_TABLE_CUSTOM=y -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4mb.csv" -CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y -CONFIG_SPIRAM=y -CONFIG_SPIRAM_IGNORE_NOTFOUND=y -CONFIG_SPIRAM_USE_MALLOC=y -CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=16384 -CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=32768 -CONFIG_ESP_TASK_WDT_TIMEOUT_S=30 diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 deleted file mode 100644 index e2940388858..00000000000 --- a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32 +++ /dev/null @@ -1,5 +0,0 @@ -CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y -CONFIG_ESPTOOLPY_FLASHFREQ_40M=y -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_4mb.csv" -CONFIG_SPIRAM_TYPE_AUTO=y -CONFIG_SPIRAM_SPEED_40M=y diff --git a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 b/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 deleted file mode 100644 index 4d724480d1a..00000000000 --- a/examples/espressif/espnn_runner/sdkconfig.defaults.esp32s3 +++ /dev/null @@ -1,9 +0,0 @@ -CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y -CONFIG_ESPTOOLPY_FLASHMODE_QIO=y -CONFIG_ESPTOOLPY_FLASHFREQ_80M=y -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_16mb.csv" -CONFIG_SPIRAM_MODE_OCT=y -CONFIG_SPIRAM_SPEED_80M=y -CONFIG_ESP32S3_DATA_CACHE_64KB=y -CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y -CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y diff --git a/examples/espressif/espnn_runner/sdkconfig.emulator b/examples/espressif/espnn_runner/sdkconfig.emulator deleted file mode 100644 index 75f258b4983..00000000000 --- a/examples/espressif/espnn_runner/sdkconfig.emulator +++ /dev/null @@ -1 +0,0 @@ -CONFIG_NN_ANSI_C=y diff --git a/examples/espressif/espnn_runner/sdkconfig.executorch b/examples/espressif/espnn_runner/sdkconfig.executorch deleted file mode 100644 index 2e0962c8be4..00000000000 --- a/examples/espressif/espnn_runner/sdkconfig.executorch +++ /dev/null @@ -1 +0,0 @@ -CONFIG_ESPNN_EXECUTORCH_DELEGATE=y diff --git a/examples/espressif/executor_runner/CMakeLists.txt b/examples/espressif/executor_runner/CMakeLists.txt index e353c3913b7..2551b1f26a0 100644 --- a/examples/espressif/executor_runner/CMakeLists.txt +++ b/examples/espressif/executor_runner/CMakeLists.txt @@ -147,6 +147,11 @@ if(ESP_PLATFORM) list(APPEND esp_runner_libs xnnpack_backend) endif() + # quantize/dequantize kernels for quantized models with CPU regions + if(TARGET quantized_ops_lib) + list(APPEND esp_runner_libs quantized_ops_lib) + endif() + if(EXECUTORCH_ENABLE_EVENT_TRACER) target_compile_definitions(${COMPONENT_LIB} PUBLIC ET_EVENT_TRACER_ENABLED) list(APPEND esp_runner_libs etdump flatccrt) diff --git a/examples/espressif/export_espnn_test_programs.py b/examples/espressif/export_espnn_test_programs.py new file mode 100644 index 00000000000..b9133dd82c6 --- /dev/null +++ b/examples/espressif/export_espnn_test_programs.py @@ -0,0 +1,155 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Export the ESP-NN backend's test models as BundleIO programs (.bpte) for the +ESP32 executor runner (examples/espressif/project with -DET_ESPNN_BACKEND=ON). + +Each program bundles one input set and the expected outputs from the host +reference (ProgramEmulator), which reproduces the delegate bit-exactly. +Fully delegated programs must therefore match to float precision; programs +with CPU regions get a tolerance for the portable kernels' float rounding. + +Writes, into --output-dir: + .bpte one program per case + manifest.json per-program atol/rtol, delegate and CPU operator counts + operators.txt the CPU operators the programs need, for + EXECUTORCH_SELECT_OPS_LIST + + python examples/espressif/export_espnn_test_programs.py --output-dir espnn_tests +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import torch + +# The test models are loaded from the checkout: a non-editable install ships +# only the test modules that other shipped code imports. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from backends.espressif.test import ( # noqa: E402 + test_cpu_fallback as fallback_cases, + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) + +from executorch.backends.espressif import to_espnn_edge # noqa: E402 +from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 +from executorch.devtools.bundled_program.config import ( # noqa: E402 + MethodTestCase, + MethodTestSuite, +) +from executorch.devtools.bundled_program.core import BundledProgram # noqa: E402 +from executorch.devtools.bundled_program.serialize import ( # noqa: E402 + serialize_from_bundled_program_to_flatbuffer, +) +from executorch.exir.lowered_backend_module import LoweredBackendModule # noqa: E402 + +EXACT_TOLERANCE = 1e-5 +FALLBACK_TOLERANCE = 0.01 + + +def cases(): + yield "conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + yield "concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)] + yield "se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] + for name, model, shapes, _ in op_cases.CASES: + yield name, model, shapes + for name, model, shapes, _ in fallback_cases.CASES: + yield name, model, shapes + + +def as_list(value): + return list(value) if isinstance(value, (tuple, list)) else [value] + + +def export_case(name, model, shapes, seed): + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program) + exported = edge.exported_program() + generator = torch.Generator().manual_seed(seed) + inputs = [torch.randn(shape, generator=generator) for shape in shapes] + expected = [ + t.to(torch.float32) for t in as_list(ProgramEmulator(exported)(*inputs)) + ] + + executorch_program = edge.to_executorch() + plan = executorch_program.executorch_program.execution_plan[0] + operators = sorted({f"{op.name}.{op.overload}" for op in plan.operators}) + bundle = BundledProgram( + executorch_program, + [MethodTestSuite("forward", [MethodTestCase(inputs, expected)])], + ) + + exact = not report["cpu_ops"] + if exact: + atol = rtol = EXACT_TOLERANCE + else: + peak = max(float(t.abs().max()) for t in expected) + atol, rtol = FALLBACK_TOLERANCE * max(peak, 1.0), FALLBACK_TOLERANCE + delegates = sum( + isinstance(module, LoweredBackendModule) + for _, module in exported.graph_module.named_children() + ) + entry = { + "name": name, + "file": f"{name}.bpte", + "atol": float(f"{atol:.6g}"), + "rtol": rtol, + "exact": exact, + "delegates": delegates, + "delegated_ops": report["delegated_ops"], + "cpu_ops": report["cpu_ops"], + "operators": operators, + } + return serialize_from_bundled_program_to_flatbuffer(bundle), entry + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument( + "--only", help="comma separated case names (default: every case)" + ) + args = parser.parse_args() + selected = set(args.only.split(",")) if args.only else None + torch.set_grad_enabled(False) + args.output_dir.mkdir(parents=True, exist_ok=True) + + manifest, operators = [], set() + for name, model, shapes in cases(): + if selected is not None and name not in selected: + continue + if any(entry["name"] == name for entry in manifest): + raise SystemExit(f"duplicate case name: {name}") + blob, entry = export_case(name, model, shapes, args.seed) + (args.output_dir / entry["file"]).write_bytes(blob) + manifest.append(entry) + operators.update(entry["operators"]) + print( + f"{name:34s} delegates={entry['delegates']} " + f"cpu_ops={len(entry['cpu_ops'])} atol={entry['atol']} " + f"({len(blob)} bytes)" + ) + if selected is not None and len(manifest) != len(selected): + missing = selected - {entry["name"] for entry in manifest} + raise SystemExit(f"unknown case names: {sorted(missing)}") + + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) + (args.output_dir / "operators.txt").write_text(",".join(sorted(operators)) + "\n") + print(f"wrote {len(manifest)} programs to {args.output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/project/CMakeLists.txt b/examples/espressif/project/CMakeLists.txt index 13a303ffd72..a43760c33c4 100644 --- a/examples/espressif/project/CMakeLists.txt +++ b/examples/espressif/project/CMakeLists.txt @@ -23,5 +23,13 @@ set(EXECUTORCH_ROOT # Add the executor_runner as an extra component set(EXTRA_COMPONENT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/../executor_runner") +# Register the ESP-NN delegate (backends/espressif) with the runtime +option(ET_ESPNN_BACKEND "Build in the ESP-NN ExecuTorch backend" OFF) +if(ET_ESPNN_BACKEND) + list(APPEND EXTRA_COMPONENT_DIRS + "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime" + ) +endif() + include($ENV{IDF_PATH}/tools/cmake/project.cmake) project(executorch_esp_runner) diff --git a/examples/espressif/scripts/bench_models.py b/examples/espressif/scripts/bench_models.py deleted file mode 100644 index b63b60f219b..00000000000 --- a/examples/espressif/scripts/bench_models.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Model factories for examples/espressif/scripts/benchmark.py. - -Use with aot_espnn_compiler.py as --model examples/espressif/scripts/bench_models.py:. -Small models use fixed-seed weights: they measure latency and memory, not accuracy. -""" - -from __future__ import annotations - -import torch -import torch.nn as nn -import torch.nn.functional as F - - -def _randomize_batch_norm(model: nn.Module, seed: int) -> nn.Module: - generator = torch.Generator().manual_seed(seed) - for module in model.modules(): - if isinstance(module, nn.BatchNorm2d): - channels = module.num_features - module.running_mean.copy_(torch.randn(channels, generator=generator) * 0.1) - module.running_var.copy_( - torch.rand(channels, generator=generator) * 0.5 + 0.75 - ) - module.weight.data.copy_( - torch.rand(channels, generator=generator) * 0.5 + 0.75 - ) - module.bias.data.copy_(torch.randn(channels, generator=generator) * 0.1) - return model - - -class DSCNN(nn.Module): - """Keyword spotting DS-CNN (MLPerf Tiny style) on a 1x49x10 MFCC input.""" - - def __init__(self, num_classes: int = 12, width: int = 64): - super().__init__() - layers = [ - nn.Conv2d(1, width, (10, 4), stride=(2, 2), padding=(5, 1), bias=False), - nn.BatchNorm2d(width), - nn.ReLU(), - ] - for _ in range(4): - layers += [ - nn.Conv2d(width, width, 3, padding=1, groups=width, bias=False), - nn.BatchNorm2d(width), - nn.ReLU(), - nn.Conv2d(width, width, 1, bias=False), - nn.BatchNorm2d(width), - nn.ReLU(), - ] - self.features = nn.Sequential(*layers) - self.pool = nn.AdaptiveAvgPool2d((1, 1)) - self.classifier = nn.Linear(width, num_classes) - - def forward(self, x): - return self.classifier(torch.flatten(self.pool(self.features(x)), 1)) - - -class BasicBlock(nn.Module): - def __init__(self, channels: int): - super().__init__() - self.conv1 = nn.Conv2d(channels, channels, 3, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(channels) - self.conv2 = nn.Conv2d(channels, channels, 3, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(channels) - - def forward(self, x): - y = F.relu(self.bn1(self.conv1(x))) - return F.relu(self.bn2(self.conv2(y)) + x) - - -class TinyResNet(nn.Module): - """64x64 image classifier with residuals, grouped conv and concat.""" - - def __init__(self, num_classes: int = 10): - super().__init__() - self.stem = nn.Sequential( - nn.Conv2d(3, 24, 3, stride=2, padding=1, bias=False), - nn.BatchNorm2d(24), - nn.ReLU(), - ) - self.pool = nn.MaxPool2d(3, stride=2, padding=1) - self.block1 = BasicBlock(24) - self.down = nn.Sequential( - nn.Conv2d(24, 48, 3, stride=2, padding=1, groups=4, bias=False), - nn.BatchNorm2d(48), - nn.ReLU6(), - ) - self.branch_a = nn.Sequential(nn.Conv2d(48, 32, 1), nn.ReLU()) - self.branch_b = nn.Sequential(nn.Conv2d(48, 32, 3, padding=1), nn.ReLU()) - self.block2 = BasicBlock(64) - self.avg = nn.AvgPool2d(2) - self.head = nn.Sequential( - nn.Flatten(), - nn.Linear(64 * 4 * 4, 128), - nn.ReLU(), - nn.Linear(128, num_classes), - ) - - def forward(self, x): - x = self.block1(self.pool(self.stem(x))) - x = self.down(x) - x = self.block2(torch.cat([self.branch_a(x), self.branch_b(x)], dim=1)) - return self.head(self.avg(x)) - - -class TinyTransformer(nn.Module): - """Patch-embedding transformer: shows the cost of CPU fallback (LayerNorm, attention).""" - - def __init__(self, dim: int = 32, heads: int = 2, classes: int = 10): - super().__init__() - self.heads = heads - self.stem = nn.Conv2d(3, dim, 8, stride=8) - self.norm1 = nn.LayerNorm(dim) - self.qkv = nn.Linear(dim, 3 * dim) - self.proj = nn.Linear(dim, dim) - self.norm2 = nn.LayerNorm(dim) - self.mlp1 = nn.Linear(dim, 2 * dim) - self.mlp2 = nn.Linear(2 * dim, dim) - self.head = nn.Linear(dim, classes) - - def forward(self, x): - tokens = self.stem(x).flatten(2).transpose(1, 2) - n, t, d = tokens.shape - q, k, v = self.qkv(self.norm1(tokens)).chunk(3, dim=-1) - q, k, v = ( - z.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) - for z in (q, k, v) - ) - attention = torch.softmax( - q @ k.transpose(-2, -1) / (d // self.heads) ** 0.5, dim=-1 - ) - tokens = tokens + self.proj((attention @ v).transpose(1, 2).reshape(n, t, d)) - tokens = tokens + self.mlp2(F.gelu(self.mlp1(self.norm2(tokens)))) - return self.head(tokens.mean(dim=1)) - - -class SegmentationOut(nn.Module): - def __init__(self, model): - super().__init__() - self.model = model - - def forward(self, x): - return self.model(x)["out"] - - -class DetectorHeads(nn.Module): - """SSDLite backbone and heads; anchor decoding and NMS stay on the host or app side.""" - - def __init__(self, model): - super().__init__() - self.backbone = model.backbone - self.head = model.head - - def forward(self, x): - head = self.head(list(self.backbone(x).values())) - return head["bbox_regression"], head["cls_logits"] - - -def ds_cnn_kws() -> nn.Module: - torch.manual_seed(1) - return _randomize_batch_norm(DSCNN(), 1).eval() - - -def tiny_resnet() -> nn.Module: - torch.manual_seed(0) - return _randomize_batch_norm(TinyResNet(), 0).eval() - - -def tiny_transformer() -> nn.Module: - torch.manual_seed(2) - return TinyTransformer().eval() - - -def lraspp_mobilenet_v3() -> nn.Module: - import torchvision - - return SegmentationOut( - torchvision.models.segmentation.lraspp_mobilenet_v3_large(weights="DEFAULT") - ).eval() - - -def ssdlite320_mobilenet_v3() -> nn.Module: - import torchvision - - return DetectorHeads( - torchvision.models.detection.ssdlite320_mobilenet_v3_large(weights="DEFAULT") - ).eval() diff --git a/examples/espressif/scripts/benchmark.py b/examples/espressif/scripts/benchmark.py deleted file mode 100644 index d87e4a195b5..00000000000 --- a/examples/espressif/scripts/benchmark.py +++ /dev/null @@ -1,526 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Benchmark the ESP-NN backend on a connected Espressif board. - -For each model in the zoo this script quantizes and lowers it with -aot_espnn_compiler.py, packs the results into bundles that fit the detected -chip's flash, builds and flashes examples/espressif/espnn_runner, and records -correctness (bit-exact or SQNR against the host) and latency (min/mean/max over -timed iterations). Results go to /.json and .md. - -Run inside an activated ESP-IDF shell with a Python that has executorch: - python examples/espressif/scripts/benchmark.py --port /dev/ttyACM0 \ - --images path/to/imagenette2-160 --dashboard - -Models that need more RAM than the board has are reported as "did not fit". -ExecuTorch-mode models need ExecuTorch cross-compiled for the target -(build_executorch_esp.sh) and enough flash for the runtime (about 6 MB with -all portable kernels); pass --skip-executorch on 4 MB boards. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -import time -from dataclasses import dataclass -from pathlib import Path - -SCRIPTS_DIR = Path(__file__).resolve().parent -EXAMPLES_DIR = SCRIPTS_DIR.parent -sys.path.insert(0, str(SCRIPTS_DIR)) -sys.path.insert(0, str(EXAMPLES_DIR)) - -import espnn_board as board # noqa: E402 -from espnn_bundle import read_bundle, write_bundle # noqa: E402 - -ZOO = f"{SCRIPTS_DIR / 'bench_models.py'}" -IMAGENET = (["--mean", "0.485", "0.456", "0.406"], ["--std", "0.229", "0.224", "0.225"]) -FIRMWARE_ESTIMATE = {"standalone": 600 * 1024, "executorch": 7 * 1024 * 1024} - - -@dataclass -class BenchModel: - name: str - model: str - shapes: list[str] - mode: str - images: bool - description: str - extra: tuple[str, ...] = () - - -MODELS = [ - BenchModel( - "ds_cnn_kws", - f"{ZOO}:ds_cnn_kws", - ["1,1,49,10"], - "standalone", - False, - "Keyword spotting DS-CNN, 49x10 MFCC", - ), - BenchModel( - "tiny_resnet_64", - f"{ZOO}:tiny_resnet", - ["1,3,64,64"], - "standalone", - True, - "Small residual CNN, 64x64", - ), - BenchModel( - "mobilenet_v3_small_96", - "mobilenet_v3_small", - ["1,3,96,96"], - "standalone", - True, - "MobileNetV3-Small, ImageNet, 96x96", - ("--resize", "110"), - ), - BenchModel( - "mobilenet_v3_small_224", - "mobilenet_v3_small", - ["1,3,224,224"], - "standalone", - True, - "MobileNetV3-Small, ImageNet, 224x224", - ), - BenchModel( - "mobilenet_v2_224", - "mobilenet_v2", - ["1,3,224,224"], - "standalone", - True, - "MobileNetV2, ImageNet, 224x224", - ), - BenchModel( - "mobilenet_v3_large_224", - "mobilenet_v3_large", - ["1,3,224,224"], - "standalone", - True, - "MobileNetV3-Large, ImageNet, 224x224", - ), - BenchModel( - "efficientnet_b0_224", - "efficientnet_b0", - ["1,3,224,224"], - "standalone", - True, - "EfficientNet-B0, ImageNet, 224x224", - ), - BenchModel( - "lraspp_mobilenet_v3_seg", - f"{ZOO}:lraspp_mobilenet_v3", - ["1,3,224,224"], - "executorch", - True, - "LR-ASPP MobileNetV3 segmentation, 21 classes, 224x224", - ("--crc-outputs",), - ), - BenchModel( - "ssdlite320_mobilenet_v3_det", - f"{ZOO}:ssdlite320_mobilenet_v3", - ["1,3,320,320"], - "executorch", - True, - "SSDLite320 MobileNetV3 detection heads, COCO, 320x320", - ("--crc-outputs", "--mean", "0.5", "0.5", "0.5", "--std", "0.5", "0.5", "0.5"), - ), - BenchModel( - "tiny_transformer_fallback", - f"{ZOO}:tiny_transformer", - ["1,3,64,64"], - "executorch", - False, - "Small vision transformer: LayerNorm and attention on the CPU fallback", - ), -] - - -def compile_model(spec: BenchModel, out_dir: Path, args) -> Path: - suffix = ".xtp" if spec.mode == "executorch" else ".xtb" - bundle = out_dir / f"{spec.name}{suffix}" - if ( - bundle.exists() - and bundle.with_suffix(".json").exists() - and not args.rebuild_models - ): - return bundle - command = [ - sys.executable, - str(EXAMPLES_DIR / "aot_espnn_compiler.py"), - "--model", - spec.model, - "--mode", - spec.mode, - "--output", - str(bundle), - "--name", - spec.name, - "--calib-samples", - str(args.calib_samples), - "--eval-samples", - str(args.eval_samples), - "--vector-samples", - "2", - ] - for shape in spec.shapes: - command += ["--input-shape", shape] - if spec.images and args.images: - command += [ - "--calib-dir", - str(Path(args.images) / "train"), - "--eval-dir", - str(Path(args.images) / "val"), - ] - if not any(flag in spec.extra for flag in ("--mean",)) and spec.images: - command += IMAGENET[0] + IMAGENET[1] - command += list(spec.extra) - print(f"[compile] {spec.name}", flush=True) - log = out_dir / f"{spec.name}.compile.log" - result = subprocess.run(command, capture_output=True, text=True) - log.write_text(result.stdout + result.stderr) - if result.returncode != 0: - raise RuntimeError(f"compiling {spec.name} failed; see {log}") - return bundle - - -def group_bundles(entries, budget: int): - groups, current, size = [], [], 0 - for entry in sorted(entries, key=lambda e: len(e[1]) + len(e[2])): - entry_size = len(entry[1]) + len(entry[2]) + 256 - if entry_size > budget: - yield None, entry - continue - if current and size + entry_size > budget: - groups.append(current) - current, size = [], 0 - current.append(entry) - size += entry_size - if current: - groups.append(current) - for group in groups: - yield group, None - - -def markdown(target: str, chip: dict, rows: list[dict], calib_samples: int) -> str: - lines = [ - f"# ESP-NN backend benchmarks: {target}", - "", - f"Board: {chip.get('target', target)}, {chip.get('cores', '?')} cores at " - f"{chip.get('cpu_mhz', '?')} MHz, {chip.get('flash_mb', '?')} MB flash, " - f"{chip.get('psram_kb', '?')} KB PSRAM, ESP-IDF {chip.get('idf', '?')}.", - "", - "| Model | Input | Path | Operators | Model size | Arena | Latency mean (min) | FPS | On-board check | Host accuracy |", - "|---|---|---|---|---|---|---|---|---|---|", - ] - for row in rows: - lines.append( - "| " - + " | ".join( - str(row.get(k, "")) - for k in ( - "description", - "input", - "path", - "operators", - "size", - "arena", - "latency", - "fps", - "check", - "accuracy", - ) - ) - + " |" - ) - lines.append("") - lines.append( - "Latency is the model invocation only (no input preprocessing), timed over " - "repeated runs after a bit-exact (or SQNR) check against the host reference." - ) - lines.append("") - lines.append( - f"Host accuracy is the int8 model on the host (the board's output is bit-identical) " - f"after post-training quantization with {calib_samples} calibration images and no " - "QAT, on 32 Imagenette validation images mapped to ImageNet classes. Models with " - "random weights report top-1 agreement or SQNR against the float model instead." - ) - return "\n".join(lines) + "\n" - - -def summarize( - spec: BenchModel, - report: dict, - case: dict | None, - failure: str | None, - board_chip: dict, -) -> dict: - row = { - "name": spec.name, - "description": spec.description, - "input": " / ".join(spec.shapes), - } - if spec.mode == "standalone": - row["path"] = "standalone XTS3" - row["operators"] = f"{sum(report.get('operations', {}).values())} int8" - row["size"] = f"{report.get('blob_bytes', 0) / 1e6:.2f} MB" - else: - row["path"] = "ExecuTorch .pte" - row["operators"] = ( - f"{report.get('delegated_ops', 0)} delegated, " - f"{len(report.get('cpu_ops', []))} CPU" - ) - row["size"] = f"{report.get('pte_bytes', 0) / 1e6:.2f} MB" - evaluation = report.get("evaluation", {}) - if evaluation.get("labelled") and not spec.model.startswith(ZOO): - row["accuracy"] = ( - f"top-1 {evaluation['device_correct']}/{evaluation['labelled']} " - f"(float {evaluation['float_correct']}), agrees with float " - f"{evaluation['top1_agree_with_float']}/{evaluation['samples']}" - ) - elif evaluation.get("samples"): - row["accuracy"] = ( - f"top-1 agrees with float {evaluation['top1_agree_with_float']}/{evaluation['samples']}" - ) - else: - row["accuracy"] = f"{report.get('sqnr_vs_float_db', 0):.1f} dB vs float" - if failure or case is None: - row.update(latency="-", fps="-", arena="-", check=failure or "not run") - return row - mean_us = float(case.get("us", 0)) - row["latency"] = ( - f"{mean_us / 1000:.1f} ms ({float(case.get('min_us', mean_us)) / 1000:.1f})" - ) - row["fps"] = f"{1e6 / mean_us:.2f}" if mean_us else "-" - row["arena"] = f"{int(case.get('arena', 0)) / 1024:.0f} KB" - arena = re.search(r"allocate (\d+)-byte activation arena", case.get("error", "")) - if case.get("status") != "PASS" and arena: - free = board_chip.get("internal_free_kb", "?") if board_chip else "?" - row.update( - latency="-", - fps="-", - arena=f"{int(arena.group(1)) / 1024:.0f} KB", - check=( - f"does not fit RAM ({free} KB free, no PSRAM)" - if board_chip.get("psram_kb") == "0" - else "does not fit RAM" - ), - ) - return row - if case.get("status") != "PASS": - row["check"] = f"FAIL ({case.get('reason', 'mismatch')})" - elif case["kind"] == "standalone": - row["check"] = f"bit-exact {case.get('exact')}/{case.get('samples')}" + ( - ", ANSI too" if case.get("ansi") == "1" else "" - ) - elif case.get("exact_required") == "1": - row["check"] = "bit-exact" - else: - row["check"] = f"{float(case.get('sqnr_db', 0)):.1f} dB vs host" - if case["kind"] == "executorch" and case.get("delegate_us"): - row[ - "latency" - ] += f"; delegate {int(case['delegate_us']) / 1000:.1f}, CPU {max(int(case['cpu_us']), 0) / 1000:.1f}" - return row - - -def main() -> int: # noqa: C901 - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--port", - default=os.environ.get("ESPPORT"), - required="ESPPORT" not in os.environ, - help="serial port of the board (default: $ESPPORT)", - ) - parser.add_argument( - "--target", help="override the detected chip (esp32s3, esp32, ...)" - ) - parser.add_argument("--models", help="comma separated subset of model names") - parser.add_argument( - "--images", help="Imagenette-style folder with train/ and val/ for calibration" - ) - parser.add_argument("--calib-samples", type=int, default=64) - parser.add_argument("--eval-samples", type=int, default=32) - parser.add_argument("--iterations", type=int, default=10) - parser.add_argument("--output", type=Path, default=EXAMPLES_DIR / "benchmarks") - parser.add_argument( - "--work-dir", type=Path, default=EXAMPLES_DIR / "benchmarks" / "work" - ) - parser.add_argument("--rebuild-models", action="store_true") - parser.add_argument("--skip-executorch", action="store_true") - parser.add_argument("--executorch-build-dir", type=Path) - parser.add_argument("--executorch-source-dir", type=Path) - parser.add_argument( - "--dashboard", - action="store_true", - help="live results at http://127.0.0.1:8770/", - ) - parser.add_argument("--timeout", type=float, default=900) - args = parser.parse_args() - - chip = board.detect_chip(args.port) - target = args.target or chip.target - print( - f"[board] {chip.description} -> target {target}, {chip.flash_mb} MB flash", - flush=True, - ) - selected = [ - m for m in MODELS if not args.models or m.name in args.models.split(",") - ] - if args.skip_executorch: - selected = [m for m in selected if m.mode != "executorch"] - - dashboard = None - if args.dashboard: - import board_dashboard - - dashboard = board_dashboard.start(open_browser=True) - - model_dir = args.work_dir / "models" - model_dir.mkdir(parents=True, exist_ok=True) - reports = {} - bundles = {"standalone": [], "executorch": []} - for spec in selected: - try: - bundle = compile_model(spec, model_dir, args) - except RuntimeError as error: - print(f"[compile] {error}", flush=True) - reports[spec.name] = {"error": str(error)} - continue - reports[spec.name] = json.loads(bundle.with_suffix(".json").read_text())[0] - _, entries = read_bundle(bundle) - bundles[spec.mode].extend(entries) - - cases: dict[str, dict] = {} - failures: dict[str, str] = {} - board_chip: dict = {} - for mode, entries in bundles.items(): - if not entries: - continue - budget = chip.app_partition_bytes - FIRMWARE_ESTIMATE[mode] - build = board.RunnerBuild( - target=target, - executorch=mode == "executorch", - build_dir=args.work_dir / f"build-{target}-{mode}", - executorch_build_dir=args.executorch_build_dir, - executorch_source_dir=args.executorch_source_dir, - ) - index = -1 - for group, too_big in group_bundles(entries, budget): - if too_big is not None: - failures[too_big[0]] = f"does not fit {chip.flash_mb} MB flash" - continue - index += 1 - magic = b"XTP1" if mode == "executorch" else b"XTB1" - path = ( - args.work_dir - / f"{target}-{mode}-{index}{'.xtp' if mode == 'executorch' else '.xtb'}" - ) - write_bundle(path, magic, group) - names = [name for name, _, _ in group] - print(f"[board] {mode} group {index}: {', '.join(names)}", flush=True) - job = f"{mode}-{index}" - if dashboard: - dashboard.begin_job( - job, - f"{target} {mode} group {index}", - {name: {"host": reports.get(name, {})} for name in names}, - ) - try: - result = board.run_bundle( - build, - path, - args.port, - args.iterations, - args.timeout, - args.work_dir / "logs" / job, - on_line=(lambda line: dashboard.log(line)) if dashboard else None, - on_case=( - (lambda case, job=job: dashboard.case(job, case)) - if dashboard - else None - ), - on_status=( - (lambda status, job=job: dashboard.status(job, status)) - if dashboard - else None - ), - ) - except RuntimeError as error: - for name in names: - failures[name] = str(error).splitlines()[0] - if dashboard: - dashboard.status(job, "build failed") - continue - board_chip = result.chip or board_chip - for case in result.cases: - cases[case.get("name", "?")] = case - for name in names: - if name not in cases: - failures[name] = ( - "did not run (out of memory or crash; see board.log)" - ) - if dashboard: - dashboard.status(job, "PASS" if result.passed else "FAIL") - - rows = [] - for spec in selected: - report = reports.get(spec.name, {}) - if "error" in report: - rows.append( - { - "name": spec.name, - "description": spec.description, - "input": " / ".join(spec.shapes), - "check": "compile failed", - } - ) - continue - rows.append( - summarize( - spec, report, cases.get(spec.name), failures.get(spec.name), board_chip - ) - ) - args.output.mkdir(parents=True, exist_ok=True) - payload = { - "target": target, - "chip": board_chip or {"target": target}, - "esptool": chip.description, - "date": time.strftime("%Y-%m-%d"), - "iterations": args.iterations, - "rows": rows, - "raw_cases": cases, - "failures": failures, - } - (args.output / f"{target}.json").write_text(json.dumps(payload, indent=2)) - (args.output / f"{target}.md").write_text( - markdown(target, payload["chip"], rows, args.calib_samples) - ) - print((args.output / f"{target}.md").read_text()) - if dashboard: - dashboard.finish() - print("dashboard still serving at http://127.0.0.1:8770/ (Ctrl+C to stop)") - try: - while True: - time.sleep(3600) - except KeyboardInterrupt: - pass - return 0 if all(not failures.get(spec.name) for spec in selected) else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/espressif/scripts/board_dashboard.py b/examples/espressif/scripts/board_dashboard.py deleted file mode 100644 index a24ad74dee1..00000000000 --- a/examples/espressif/scripts/board_dashboard.py +++ /dev/null @@ -1,308 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Live Chrome dashboard for on-board runs of examples/espressif/espnn_runner. - -Used by benchmark.py --dashboard; can also show a saved results file: - python examples/espressif/scripts/board_dashboard.py --results benchmarks/esp32s3.json -""" - -from __future__ import annotations - -import argparse -import http.server -import json -import socketserver -import threading -import time -import webbrowser -from pathlib import Path - - -class State: - def __init__(self): - self.lock = threading.Lock() - self.changed = threading.Condition(self.lock) - self.version = 0 - self.jobs: dict[str, dict] = {} - self.order: list[str] = [] - self.logs: list[str] = [] - self.running = False - - def touch(self): - self.version += 1 - self.changed.notify_all() - - def snapshot(self) -> dict: - with self.lock: - return { - "jobs": self.jobs, - "logs": self.logs[-200:], - "running": self.running, - "version": self.version, - "order": list(self.order), - } - - -class Dashboard: - """Thread-safe facade that benchmark.py calls while a run progresses.""" - - def __init__(self, state: State, url: str): - self.state = state - self.url = url - with state.lock: - state.running = True - state.touch() - - def begin_job(self, job: str, title: str, cases: dict) -> None: - with self.state.lock: - self.state.jobs[job] = { - "title": title, - "status": "queued", - "cases": cases, - "result_line": "", - } - if job not in self.state.order: - self.state.order.append(job) - self.state.touch() - - def status(self, job: str, status: str) -> None: - with self.state.lock: - self.state.jobs.setdefault(job, {"cases": {}})["status"] = status - self.state.touch() - - def case(self, job: str, fields: dict) -> None: - with self.state.lock: - cases = self.state.jobs.setdefault(job, {"cases": {}}).setdefault( - "cases", {} - ) - cases.setdefault(fields.get("name", "?"), {})["device"] = fields - self.state.touch() - - def log(self, line: str) -> None: - if not line.startswith( - ("SUITE", "ET ", "CHIP", "ESPNN", "ERROR", "Guru", "abort") - ): - return - with self.state.lock: - self.state.logs.append(line[:240]) - del self.state.logs[:-400] - self.state.touch() - - def finish(self) -> None: - with self.state.lock: - self.state.running = False - self.state.touch() - - -PAGE = r""" - - -ESP-NN Board Dashboard - - -

ESP-NN board dashboard

idle
-
-

Live log

- -""" - - -def make_handler(state: State): - class Handler(http.server.BaseHTTPRequestHandler): - protocol_version = "HTTP/1.1" - - def log_message(self, *args): - pass - - def do_GET(self): - path = self.path.split("?", 1)[0] - try: - if path == "/": - body = PAGE.encode() - self.send_response(200) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - elif path == "/state.json": - body = json.dumps(state.snapshot()).encode() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Cache-Control", "no-store") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - elif path == "/events": - self.send_response(200) - self.send_header("Content-Type", "text/event-stream") - self.send_header("Cache-Control", "no-store") - self.send_header("Connection", "close") - self.end_headers() - seen = -1 - while True: - with state.changed: - state.changed.wait_for( - lambda seen=seen: state.version != seen, timeout=10 - ) - snapshot = state.snapshot() - seen = snapshot["version"] - self.wfile.write( - b"data: " + json.dumps(snapshot).encode() + b"\n\n" - ) - self.wfile.flush() - else: - self.send_response(404) - self.send_header("Content-Length", "0") - self.end_headers() - except (BrokenPipeError, ConnectionResetError): - pass - - return Handler - - -class Server(socketserver.ThreadingMixIn, http.server.HTTPServer): - daemon_threads = True - allow_reuse_address = True - - -def start( - port: int = 8770, open_browser: bool = False, state: State | None = None -) -> Dashboard: - state = state or State() - server = Server(("127.0.0.1", port), make_handler(state)) - threading.Thread(target=server.serve_forever, daemon=True).start() - url = f"http://127.0.0.1:{port}/" - print(f"dashboard: {url}", flush=True) - if open_browser: - webbrowser.open(url) - return Dashboard(state, url) - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument( - "--results", type=Path, required=True, help="benchmark.py JSON results" - ) - parser.add_argument("--port", type=int, default=8770) - args = parser.parse_args() - results = json.loads(args.results.read_text()) - dashboard = start(args.port, open_browser=True) - title = f"{results.get('target', '')} benchmark {results.get('date', '')}" - dashboard.begin_job("results", title, {}) - for case in results.get("raw_cases", {}).values(): - dashboard.case("results", case) - dashboard.status("results", "loaded") - dashboard.finish() - try: - while True: - time.sleep(3600) - except KeyboardInterrupt: - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/espressif/scripts/build_executorch_esp.sh b/examples/espressif/scripts/build_executorch_esp.sh deleted file mode 100755 index 6d8c79e5074..00000000000 --- a/examples/espressif/scripts/build_executorch_esp.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -# Cross-compile the ExecuTorch runtime, portable kernels and quantized kernels -# for an Espressif chip with the ESP-IDF toolchain. -# -# Run inside an activated ESP-IDF shell (. $IDF_PATH/export.sh): -# examples/espressif/scripts/build_executorch_esp.sh [esp32s3|esp32|esp32s2|esp32p4|...] -# -# Environment: -# EXECUTORCH_ROOT ExecuTorch source tree (default: this repository) -# ET_BUILD_DIR output directory (default: $EXECUTORCH_ROOT/cmake-out-esp-) -# CMAKE CMake >= 3.24 (ExecuTorch requirement) -# PYTHON_EXECUTABLE Python with executorch and torch installed (kernel registration codegen) -# ET_SELECT_OPS_LIST comma separated ops for a selective build, e.g. "aten::add.out,..." -# ET_SELECT_OPS_MODEL a .pte whose operators define the selective build -set -euo pipefail - -TARGET=${1:-${IDF_TARGET:-esp32s3}} -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) -EXECUTORCH_ROOT=${EXECUTORCH_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)} -BUILD_DIR=${ET_BUILD_DIR:-$EXECUTORCH_ROOT/cmake-out-esp-$TARGET} -CMAKE=${CMAKE:-cmake} -PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} - -if [ -z "${IDF_PATH:-}" ]; then - echo "build_executorch_esp: IDF_PATH is not set; run . \$IDF_PATH/export.sh first" >&2 - exit 1 -fi - -case "$TARGET" in - esp32|esp32s2|esp32s3) - TARGET_FLAGS="-mlongcalls -fno-builtin-memcpy -fno-builtin-memset -fno-builtin-bzero -mdisable-hardware-atomics" ;; - *) - TARGET_FLAGS="-fno-builtin-memcpy -fno-builtin-memset -fno-builtin-bzero" ;; -esac - -SELECT_ARGS=() -if [ -n "${ET_SELECT_OPS_LIST:-}" ]; then - SELECT_ARGS+=(-DEXECUTORCH_SELECT_OPS_LIST="$ET_SELECT_OPS_LIST") -fi -if [ -n "${ET_SELECT_OPS_MODEL:-}" ]; then - SELECT_ARGS+=(-DEXECUTORCH_SELECT_OPS_MODEL="$ET_SELECT_OPS_MODEL") -fi -if [ ${#SELECT_ARGS[@]} -eq 0 ]; then - SELECT_ARGS+=(-DEXECUTORCH_BUILD_PORTABLE_OPS=ON) -fi - -mkdir -p "$BUILD_DIR" -cd "$EXECUTORCH_ROOT" -"$CMAKE" --preset esp-baremetal -B "$BUILD_DIR" -G Ninja \ - -DCMAKE_TOOLCHAIN_FILE="$IDF_PATH/tools/cmake/toolchain-$TARGET.cmake" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \ - -DCMAKE_INSTALL_PREFIX="$BUILD_DIR" \ - -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \ - -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ - -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=OFF \ - -DEXECUTORCH_ENABLE_LOGGING=ON \ - -DEXECUTORCH_BUILD_DEVTOOLS=OFF \ - -DEXECUTORCH_BUILD_PTHREADPOOL=OFF \ - -DEXECUTORCH_BUILD_CPUINFO=OFF \ - -DEXECUTORCH_BUILD_XNNPACK=OFF \ - -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=OFF \ - -DEXECUTORCH_PAL_DEFAULT=minimal \ - "${SELECT_ARGS[@]}" - -# Outside an ESP-IDF project the toolchain file leaves its flag response -# files empty; ESP32/S2/S3 code needs -mlongcalls to link. -for flags_file in cflags cxxflags asmflags; do - printf '%s\n' $TARGET_FLAGS > "$BUILD_DIR/toolchain/$flags_file" -done -"$CMAKE" --build "$BUILD_DIR" -j"$(nproc)" --target install -echo "build_executorch_esp: libraries in $BUILD_DIR/lib, headers in $BUILD_DIR/include" diff --git a/examples/espressif/scripts/espnn_board.py b/examples/espressif/scripts/espnn_board.py deleted file mode 100644 index 46975c87788..00000000000 --- a/examples/espressif/scripts/espnn_board.py +++ /dev/null @@ -1,300 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Board helpers for the ESP-NN examples: chip detection, ESP-IDF build and -flash of examples/espressif/espnn_runner, serial capture with reset, and -parsing of the runner's result lines. Needs an activated ESP-IDF shell. -""" - -from __future__ import annotations - -import fcntl -import os -import re -import select -import struct -import subprocess -import sys -import termios -import time -import tty -from dataclasses import dataclass, field -from pathlib import Path - -EXAMPLES_DIR = Path(__file__).resolve().parents[1] -RUNNER_DIR = EXAMPLES_DIR / "espnn_runner" -EXECUTORCH_ROOT = EXAMPLES_DIR.parents[1] -TIOCM_DTR = 0x002 -TIOCM_RTS = 0x004 -FIELDS = re.compile(r"(\w+)=(\S+)") -APP_PARTITION_BYTES = {4: 0x3F0000, 8: 0x7F0000, 16: 15 * 1024 * 1024} - - -def idf_command() -> list[str]: - idf_path = os.environ.get("IDF_PATH") - if not idf_path: - raise RuntimeError("IDF_PATH is not set; run . $IDF_PATH/export.sh first") - python = os.environ.get("IDF_PYTHON_ENV_PATH") - interpreter = f"{python}/bin/python" if python else "python3" - return [interpreter, f"{idf_path}/tools/idf.py"] - - -def tool_env() -> dict[str, str]: - """ESP-IDF tools run in their own venv; a PYTHONPATH pointing at the torch - environment would shadow its pinned packages (click, esptool, ...).""" - return { - k: v for k, v in os.environ.items() if k not in ("PYTHONPATH", "PYTHONHOME") - } - - -def esptool_command() -> list[str]: - python = os.environ.get("IDF_PYTHON_ENV_PATH") - return [f"{python}/bin/python" if python else "python3", "-m", "esptool"] - - -@dataclass -class ChipInfo: - target: str - description: str - flash_mb: int - - @property - def app_partition_bytes(self) -> int: - return APP_PARTITION_BYTES.get(self.flash_mb, APP_PARTITION_BYTES[4]) - - -def detect_chip(port: str) -> ChipInfo: - output = subprocess.run( - esptool_command() + ["--port", port, "flash_id"], - capture_output=True, - text=True, - timeout=60, - env=tool_env(), - ).stdout - match = re.search(r"Chip (?:is|type:)\s*(ESP32[-\w]*)([^\n]*)", output) - if not match: - raise RuntimeError(f"Could not identify the chip on {port}:\n{output[-800:]}") - name = match.group(1).upper() - family = re.match(r"ESP32-(S2|S3|C2|C3|C5|C6|C61|H2|H4|P4)\b", name) - target = f"esp32{family.group(1).lower()}" if family else "esp32" - flash = re.search(r"Detected flash size:\s*(\d+)MB", output) - return ChipInfo( - target=target, - description=(match.group(1) + match.group(2)).strip(), - flash_mb=int(flash.group(1)) if flash else 4, - ) - - -def set_sdkconfig(path: Path, values: dict[str, str]) -> None: - lines = path.read_text().splitlines() if path.exists() else [] - remaining = dict(values) - output = [] - for line in lines: - key = line.split("=", 1)[0].replace("# ", "").split(" is not set")[0].strip() - if key in remaining: - output.append(f"{key}={remaining.pop(key)}") - else: - output.append(line) - output.extend(f"{key}={value}" for key, value in remaining.items()) - path.write_text("\n".join(output) + "\n") - - -@dataclass -class RunnerBuild: - target: str - executorch: bool - build_dir: Path - executorch_build_dir: Path | None = None - executorch_source_dir: Path | None = None - extra_defaults: list[Path] = field(default_factory=list) - - def cmake_args(self) -> list[str]: - defaults = [str(RUNNER_DIR / "sdkconfig.defaults")] - if self.executorch: - defaults.append(str(RUNNER_DIR / "sdkconfig.executorch")) - defaults += [str(p) for p in self.extra_defaults] - args = [ - f"-DIDF_TARGET={self.target}", - f"-DSDKCONFIG={self.build_dir}/sdkconfig", - f"-DSDKCONFIG_DEFAULTS={';'.join(defaults)}", - ] - if self.executorch and self.executorch_build_dir: - args.append(f"-DESPNN_EXECUTORCH_BUILD_DIR={self.executorch_build_dir}") - if self.executorch and self.executorch_source_dir: - args.append(f"-DESPNN_EXECUTORCH_SOURCE_DIR={self.executorch_source_dir}") - return args - - def configure(self, bundle: Path, iterations: int) -> None: - self.build_dir.mkdir(parents=True, exist_ok=True) - values = { - "CONFIG_ESPNN_RUNNER_BUNDLE": f'"{bundle.resolve()}"', - "CONFIG_ESPNN_RUNNER_BENCH_ITERATIONS": str(iterations), - } - sdkconfig = self.build_dir / "sdkconfig" - if sdkconfig.exists(): - set_sdkconfig(sdkconfig, values) - else: - overlay = self.build_dir / "sdkconfig.bundle" - overlay.write_text("".join(f"{k}={v}\n" for k, v in values.items())) - if overlay not in self.extra_defaults: - self.extra_defaults.append(overlay) - - def run_idf( - self, *actions: str, port: str | None = None, log: Path | None = None - ) -> str: - command = ( - idf_command() - + ["-C", str(RUNNER_DIR), "-B", str(self.build_dir)] - + self.cmake_args() - ) - if port: - command += ["-p", port, "-b", "921600"] - command += list(actions) - result = subprocess.run(command, capture_output=True, text=True, env=tool_env()) - output = result.stdout + result.stderr - if log: - log.parent.mkdir(parents=True, exist_ok=True) - log.write_text(output) - if result.returncode != 0: - errors = [ - line - for line in output.splitlines() - if "error" in line.lower() or "FAILED" in line - ] - errors = errors or output.strip().splitlines()[-6:] - raise RuntimeError( - f"idf.py {' '.join(actions)} failed:\n" + "\n".join(errors[-15:]) - ) - return output - - def binary_size(self) -> int: - image = self.build_dir / "espnn_runner.bin" - return image.stat().st_size if image.exists() else 0 - - -class SerialPort: - def __init__(self, path: str, baud: int = 115200): - self.fd = os.open(path, os.O_RDWR | os.O_NOCTTY) - tty.setraw(self.fd) - attributes = termios.tcgetattr(self.fd) - speed = getattr(termios, f"B{baud}") - attributes[4] = attributes[5] = speed - termios.tcsetattr(self.fd, termios.TCSANOW, attributes) - - def _lines(self, dtr: bool, rts: bool) -> None: - bits = struct.unpack( - "I", fcntl.ioctl(self.fd, termios.TIOCMGET, struct.pack("I", 0)) - )[0] - bits = (bits | TIOCM_DTR) if dtr else (bits & ~TIOCM_DTR) - bits = (bits | TIOCM_RTS) if rts else (bits & ~TIOCM_RTS) - fcntl.ioctl(self.fd, termios.TIOCMSET, struct.pack("I", bits)) - - def reset(self) -> None: - self._lines(dtr=False, rts=True) - time.sleep(0.1) - self._lines(dtr=False, rts=False) - - def read(self, timeout: float) -> bytes: - ready, _, _ = select.select([self.fd], [], [], timeout) - if not ready: - return b"" - try: - return os.read(self.fd, 65536) - except OSError: - return b"" - - def close(self) -> None: - os.close(self.fd) - - -@dataclass -class RunResult: - chip: dict = field(default_factory=dict) - cases: list[dict] = field(default_factory=list) - passed: bool = False - finished: bool = False - log: str = "" - last_error: str = "" - - -def parse_line(text: str, result: RunResult, on_case=None) -> None: - if text.startswith("CHIP "): - result.chip = dict(FIELDS.findall(text)) - elif text.startswith("ERROR:"): - result.last_error = text[len("ERROR:") :].strip() - elif text.startswith(("SUITE CASE", "ET CASE")): - case = dict(FIELDS.findall(text)) - case["kind"] = "executorch" if text.startswith("ET CASE") else "standalone" - if case.get("status") != "PASS" and result.last_error: - case["error"] = result.last_error - result.last_error = "" - result.cases.append(case) - if on_case: - on_case(case) - elif text.startswith("ESPNN RUNNER:"): - result.finished = True - result.passed = "PASS" in text - - -def capture(port: str, timeout: float, on_line=None, on_case=None) -> RunResult: - result = RunResult() - serial = SerialPort(port) - serial.reset() - deadline = time.time() + timeout - pending = b"" - raw = bytearray() - try: - while time.time() < deadline and not result.finished: - chunk = serial.read(0.5) - if not chunk: - continue - raw += chunk - pending += chunk - while b"\n" in pending: - line, pending = pending.split(b"\n", 1) - text = line.decode("utf-8", errors="replace").rstrip("\r") - parse_line(text, result, on_case) - if on_line: - on_line(text) - finally: - serial.close() - result.log = raw.decode("utf-8", errors="replace") - return result - - -def run_bundle( - build: RunnerBuild, - bundle: Path, - port: str, - iterations: int, - timeout: float, - log_dir: Path, - on_line=None, - on_case=None, - on_status=None, -) -> RunResult: - build.configure(bundle, iterations) - if on_status: - on_status("building") - build.run_idf("build", log=log_dir / "build.log") - if on_status: - on_status("flashing") - build.run_idf("flash", port=port, log=log_dir / "flash.log") - if on_status: - on_status("running on board") - result = capture(port, timeout, on_line, on_case) - (log_dir / "board.log").write_text(result.log) - return result - - -if __name__ == "__main__": - port = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("ESPPORT") - if not port: - sys.exit("usage: espnn_board.py PORT (or set ESPPORT)") - print(detect_chip(port)) diff --git a/examples/espressif/scripts/license_headers.py b/examples/espressif/scripts/license_headers.py deleted file mode 100644 index 244f5641cee..00000000000 --- a/examples/espressif/scripts/license_headers.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Add or verify the BSD-3-Clause license header on every source and build file -of the ESP-NN backend and its examples. - - python examples/espressif/scripts/license_headers.py --holder "Your Name or Company" - python examples/espressif/scripts/license_headers.py --check - -The header matches the ExecuTorch convention (contributor copyright line plus -the reference to the BSD-style LICENSE at the repository root) and adds an -SPDX identifier. Re-running with a different --holder replaces the line. -""" - -from __future__ import annotations - -import argparse -import re -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[3] -SCOPES = [ROOT / "backends" / "espressif", ROOT / "examples" / "espressif"] -DEFAULT_HOLDER = "Mwaniki Njagi" -YEAR = "2026" -HASH_STYLE = {".py", ".sh", ".cmake", ".yml", ".yaml"} -HASH_NAMES = {"CMakeLists.txt", "Kconfig", "Kconfig.projbuild"} -BLOCK_STYLE = {".c", ".cc", ".cpp", ".h", ".hpp", ".S"} -SKIP_PARTS = { - "build", - "managed_components", - "__pycache__", - "benchmarks", - "bundles", - "models", -} -MARKER = "SPDX-License-Identifier: BSD-3-Clause" -BODY = [ - "This source code is licensed under the BSD-style license found in the", - "LICENSE file in the root directory of this source tree.", -] - - -def header_lines(holder: str) -> list[str]: - return [ - f"Copyright {YEAR} {holder}.", - "All rights reserved.", - "", - *BODY, - "", - MARKER, - ] - - -def render(style: str, holder: str) -> str: - lines = header_lines(holder) - if style == "hash": - return "\n".join(f"# {line}".rstrip() for line in lines) + "\n" - return "/*\n" + "\n".join(f" * {line}".rstrip() for line in lines) + "\n */\n" - - -def style_for(path: Path) -> str | None: - if path.name in HASH_NAMES or path.suffix in HASH_STYLE: - return "hash" - if path.suffix in BLOCK_STYLE: - return "block" - return None - - -def strip_existing(text: str, style: str) -> str: - if MARKER not in text.split("\n\n", 2)[0] + text[:600]: - return text - if style == "hash": - pattern = re.compile(r"\A(#[^\n]*\n)+?# " + re.escape(MARKER) + r"\n\n?") - else: - pattern = re.compile(r"\A/\*.*?" + re.escape(MARKER) + r"\n \*/\n\n?", re.S) - return pattern.sub("", text, count=1) - - -def apply(path: Path, holder: str) -> bool: - style = style_for(path) - text = path.read_text() - shebang = "" - if text.startswith("#!"): - shebang, _, text = text.partition("\n") - shebang += "\n" - body = strip_existing(text, style) - body = body.lstrip("\n") - updated = shebang + render(style, holder) + ("\n" + body if body else "") - if updated != shebang + text: - path.write_text(updated) - return True - return False - - -def files(): - for scope in SCOPES: - for path in sorted(scope.rglob("*")): - parts = path.relative_to(scope).parts[:-1] - if path.is_file() and not any( - p in SKIP_PARTS or p.startswith("build") for p in parts - ): - if style_for(path): - yield path - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("--holder", default=DEFAULT_HOLDER) - parser.add_argument( - "--check", action="store_true", help="fail if any file lacks the header" - ) - args = parser.parse_args() - missing = [] - changed = 0 - for path in files(): - if args.check: - if MARKER not in path.read_text()[:800]: - missing.append(path) - elif apply(path, args.holder): - changed += 1 - if args.check: - for path in missing: - print(f"missing license header: {path.relative_to(ROOT)}") - print(f"{len(missing)} file(s) without a header") - return 1 if missing else 0 - print(f"updated {changed} file(s)") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/examples/espressif/scripts/make_test_bundles.py b/examples/espressif/scripts/make_test_bundles.py deleted file mode 100644 index 20987818986..00000000000 --- a/examples/espressif/scripts/make_test_bundles.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Build the on-board operator test bundles from the backend's unit-test models. - - op_suite.xtb every XTS3 opcode as a standalone int8 graph (bit-exact check) - pte_suite.xtp ExecuTorch programs: full delegation, multiple inputs and - outputs, and CPU fallback (cumsum, transformer, LSTM) - - python examples/espressif/scripts/make_test_bundles.py --output-dir examples/espressif/espnn_runner/bundles -""" - -from __future__ import annotations - -import argparse -import json -import math -import sys -from collections import Counter -from pathlib import Path - -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from espnn_bundle import sqnr_db, write_bundle, xte1_vectors, xtv2_vectors # noqa: E402 - -from executorch.backends.espressif import to_espnn_edge # noqa: E402 -from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 -from executorch.backends.espressif.serialization import ( # noqa: E402 - build_graph, - parse_graph, - serialize_graph, -) -from executorch.backends.espressif.test import ( # noqa: E402 # noqa: E402 # noqa: E402 - test_cpu_fallback as fallback_cases, - test_int8_lowering as lowering_cases, - test_int8_ops as op_cases, -) -from executorch.exir.lowered_backend_module import LoweredBackendModule # noqa: E402 - - -def standalone_cases(): - yield "conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)] - yield "concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)] - yield "se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] - for name, model, shapes, _ in op_cases.CASES: - yield name, model, shapes - - -def program_cases(): - yield "se-block-full-delegate", lowering_cases.SEBlockModel(), [ - (1, 3, 16, 16) - ], 99.0 - yield "detection-head-2-outputs", op_cases.DetectionHead(), [(1, 3, 16, 16)], 99.0 - yield "two-inputs-two-outputs", op_cases.TwoInputTwoOutput(), [ - (1, 3, 12, 12), - (1, 6), - ], 99.0 - for name, model, shapes, _ in fallback_cases.CASES: - yield name, model, shapes, 30.0 - - -def random_inputs(shapes, count, seed): - generator = torch.Generator().manual_seed(seed) - return [[torch.randn(s, generator=generator) for s in shapes] for _ in range(count)] - - -def build_op_suite(samples: int): - entries, reports = [], [] - for name, model, shapes in standalone_cases(): - _, program = op_cases.quantize(model, shapes) - blob = serialize_graph(build_graph(program)) - graph = parse_graph(blob) - vectors = xtv2_vectors( - graph, [[x.numpy() for x in s] for s in random_inputs(shapes, samples, 7)] - ) - operations = dict(Counter(op.opcode.name for op in graph.operations)) - entries.append((name, blob, vectors)) - reports.append( - { - "name": name, - "operations": operations, - "blob_bytes": len(blob), - "inputs": [list(graph.tensors[i].shape) for i in graph.inputs], - "outputs": [list(graph.tensors[i].shape) for i in graph.outputs], - } - ) - print(f"op_suite {name:34s} {sum(operations.values()):3d} ops {operations}") - return entries, reports - - -def build_pte_suite(samples: int): - entries, reports = [], [] - for name, model, shapes, min_sqnr in program_cases(): - converted, program = op_cases.quantize(model, shapes) - edge, report = to_espnn_edge(program) - exported = edge.exported_program() - emulator = ProgramEmulator(exported) - rows, worst = [], math.inf - for inputs in random_inputs(shapes, samples, 11): - outputs = emulator(*inputs) - outputs = list(outputs) if isinstance(outputs, (tuple, list)) else [outputs] - expected = converted(*inputs) - expected = ( - list(expected) if isinstance(expected, (tuple, list)) else [expected] - ) - worst = min( - [worst] - + [sqnr_db(w, g.to(torch.float32)) for w, g in zip(expected, outputs)] - ) - rows.append((inputs, outputs)) - exact = not report["cpu_ops"] - entries.append( - (name, edge.to_executorch().buffer, xte1_vectors(rows, exact, min_sqnr)) - ) - delegates = sum( - isinstance(m, LoweredBackendModule) - for _, m in exported.graph_module.named_children() - ) - reports.append( - { - "name": name, - "delegates": delegates, - "partitions": report["partitions"], - "delegated_ops": report["delegated_ops"], - "cpu_ops": report["cpu_ops"], - "exact_required": exact, - "min_sqnr_db": min_sqnr, - "host_sqnr_vs_pytorch_db": round(worst, 1), - "inputs": [list(t.shape) for t in rows[0][0]], - "outputs": [list(t.shape) for t in rows[0][1]], - } - ) - print( - f"pte_suite {name:34s} delegates={delegates} cpu_ops={len(report['cpu_ops'])}" - ) - return entries, reports - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument( - "--output-dir", - type=Path, - default=Path(__file__).resolve().parents[1] / "espnn_runner" / "bundles", - ) - parser.add_argument("--samples", type=int, default=3) - parser.add_argument( - "--suites", - default="op,pte", - help="comma separated: op (standalone op_suite.xtb), pte (pte_suite.xtp)", - ) - args = parser.parse_args() - suites = set(args.suites.split(",")) - torch.set_grad_enabled(False) - args.output_dir.mkdir(parents=True, exist_ok=True) - for suite, filename, magic, builder in ( - ("op", "op_suite.xtb", b"XTB1", build_op_suite), - ("pte", "pte_suite.xtp", b"XTP1", build_pte_suite), - ): - if suite not in suites: - continue - entries, reports = builder(args.samples) - path = args.output_dir / filename - size = write_bundle(path, magic, entries) - path.with_suffix(".json").write_text(json.dumps(reports, indent=2)) - print(f"wrote {path} ({size} bytes, {len(entries)} cases)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/espressif/scripts/run_bundle.py b/examples/espressif/scripts/run_bundle.py deleted file mode 100644 index 367d1233ed7..00000000000 --- a/examples/espressif/scripts/run_bundle.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2026 Mwaniki Njagi. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. -# -# SPDX-License-Identifier: BSD-3-Clause - -""" -Build examples/espressif/espnn_runner for the connected chip, flash a bundle -(.xtb int8 graphs or .xtp ExecuTorch programs) and report each case. - - python examples/espressif/scripts/run_bundle.py --port /dev/ttyUSB0 \ - examples/espressif/espnn_runner/bundles/op_suite.xtb -""" - -from __future__ import annotations - -import argparse -import json -import os -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -import espnn_board as board # noqa: E402 - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("bundle", type=Path) - parser.add_argument( - "--port", - default=os.environ.get("ESPPORT"), - required="ESPPORT" not in os.environ, - help="serial port of the board (default: $ESPPORT)", - ) - parser.add_argument("--target", help="override the detected chip") - parser.add_argument("--iterations", type=int, default=10) - parser.add_argument("--timeout", type=float, default=600) - parser.add_argument("--build-root", type=Path, default=board.RUNNER_DIR / "build") - parser.add_argument("--executorch-build-dir", type=Path) - parser.add_argument("--executorch-source-dir", type=Path) - parser.add_argument("--json", type=Path, help="write the parsed results here") - args = parser.parse_args() - - chip = board.detect_chip(args.port) - target = args.target or chip.target - executorch = args.bundle.read_bytes()[:4] == b"XTP1" - mode = "executorch" if executorch else "standalone" - print( - f"{chip.description}: {chip.flash_mb} MB flash; building {mode} runner for {target}", - flush=True, - ) - build = board.RunnerBuild( - target=target, - executorch=executorch, - build_dir=args.build_root / f"{target}-{mode}", - executorch_build_dir=args.executorch_build_dir, - executorch_source_dir=args.executorch_source_dir, - ) - result = board.run_bundle( - build, - args.bundle, - args.port, - args.iterations, - args.timeout, - build.build_dir / "logs", - on_line=lambda line: print(line, flush=True), - ) - if args.json: - args.json.write_text( - json.dumps( - {"chip": result.chip, "cases": result.cases, "passed": result.passed}, - indent=2, - ) - ) - print(f"RESULT: {'PASS' if result.passed else 'FAIL'} ({len(result.cases)} cases)") - return 0 if result.passed else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/examples/espressif/test_emulator.py b/examples/espressif/test_emulator.py index cf0d30b2972..bcdee998c4a 100644 --- a/examples/espressif/test_emulator.py +++ b/examples/espressif/test_emulator.py @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Check the emulator log from the ESP32-S3 add/multiply smoke test.""" +"""Check an ESP32-S3 emulator log from a BundleIO (.bpte) runner build.""" import argparse import re @@ -33,7 +33,7 @@ def main() -> None: if marker not in output: raise SystemExit(f"FAIL: Missing {marker!r}; see {args.log}") print( - f"PASS: ESP32-S3 ran {finished[1]} inferences and matched both PyTorch outputs." + f"PASS: ESP32-S3 ran {finished[1]} inferences and matched the bundled reference outputs." ) From ef36817bb145a5a556f881a394376ba365bac278 Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Tue, 27 Oct 2026 14:26:43 +0100 Subject: [PATCH 07/10] Fix dropout lowering, reject unwritten reads, speed up input quantization Share qparams through dropout/dropout_/alias and collapse the q-dq-q-dq chain that eval-mode dropout leaves, which fixes MobileNetV2/V3 classifiers reading an unwritten tensor. serialize_graph now rejects reads of unwritten activations. The delegate quantizes float inputs without nearbyintf, which is a software routine on Xtensa (about 93 ms down to 33 ms on MobileNetV3-Small). --- .../espnn_runtime/src/espnn_backend.cpp | 13 +++-- .../espressif/quantizer/espnn_quantizer.py | 3 ++ backends/espressif/serialization/xts3.py | 43 ++++++++++++++- backends/espressif/test/test_contracts.py | 53 +++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 7d247f6d141..1be6cf89a5c 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -50,6 +50,12 @@ namespace { DelegateStats g_stats{}; +constexpr float kHalfEvenRoundingBias = 12582912.0f; + +float round_half_even_below_2p22(float value) { + return (value + kHalfEvenRoundingBias) - kHalfEvenRoundingBias; +} + int8_t quantize(float value, float inverse_scale, int32_t zero_point) { const float scaled = value * inverse_scale; if (!std::isfinite(scaled)) @@ -58,7 +64,8 @@ int8_t quantize(float value, float inverse_scale, int32_t zero_point) { return 127; if (scaled < -1.0e6f) return -128; - int32_t q = static_cast(std::nearbyintf(scaled)) + zero_point; + int32_t q = + static_cast(round_half_even_below_2p22(scaled)) + zero_point; if (q < -128) q = -128; if (q > 127) @@ -98,8 +105,8 @@ bool load_input( destination[i] = quantize(source[i], inverse, info.zero_point); return true; } - for (size_t c = 0; c < channels; ++c) { - for (size_t p = 0; p < plane; ++p) { + for (size_t p = 0; p < plane; ++p) { + for (size_t c = 0; c < channels; ++c) { destination[p * channels + c] = quantize(source[c * plane + p], inverse, info.zero_point); } diff --git a/backends/espressif/quantizer/espnn_quantizer.py b/backends/espressif/quantizer/espnn_quantizer.py index f7cb497119a..1bebb13eb2b 100644 --- a/backends/espressif/quantizer/espnn_quantizer.py +++ b/backends/espressif/quantizer/espnn_quantizer.py @@ -301,6 +301,9 @@ def annotate_unary_and_softmax( torch.ops.aten.upsample_bilinear2d.default, torch.ops.aten.constant_pad_nd.default, torch.ops.aten.pad.default, + torch.ops.aten.dropout.default, + torch.ops.aten.dropout_.default, + torch.ops.aten.alias.default, ) SPLIT_TARGETS = ( diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py index b9cc0fdbc28..0a99476b05e 100644 --- a/backends/espressif/serialization/xts3.py +++ b/backends/espressif/serialization/xts3.py @@ -611,8 +611,23 @@ def _get_or_create_int8( # noqa: C901 quantized_node, float(node.args[1]), int(node.args[2]) ) return self._constant_int8(node) + source = quantized_node.args[0] + if getattr( + source, "op", None + ) == "call_function" and "dequantize_per_tensor" in str(source.target): + # q(dq(x)) is left behind where an identity op (such as an + # eval-mode dropout) was decomposed away. + if (float(source.args[1]), int(source.args[2])) != ( + float(node.args[1]), + int(node.args[2]), + ): + raise ValueError( + f"{quantized_node.name} requantizes {source.name}; " + "back-to-back requantization is not supported" + ) + return self._get_or_create_int8(source, filter_layout) return self._new_int8_activation( - quantized_node.args[0], + source, float(node.args[1]), int(node.args[2]), ) @@ -1917,9 +1932,35 @@ def check_runtime_limits(graph: XTS3Graph) -> None: raise ValueError(f"int32 constant {tensor.tensor_id} is not 4-byte aligned") +def check_dataflow(graph: XTS3Graph) -> None: + """ + Raise ValueError when an operation or graph output reads an activation + that is neither a graph input nor written by an earlier operation. The + device would read whatever its arena holds there. + """ + ready = set(graph.inputs) + ready.update( + tensor.tensor_id + for tensor in graph.tensors + if tensor.flags & TENSOR_FLAG_CONSTANT + ) + for index, operation in enumerate(graph.operations): + for tensor_id in operation.inputs: + if tensor_id >= 0 and tensor_id not in ready: + raise ValueError( + f"Operation {index} ({XTS3OpCode(operation.opcode).name}) " + f"reads tensor {tensor_id}, which nothing writes before it" + ) + ready.update(operation.outputs) + for tensor_id in graph.outputs: + if tensor_id not in ready: + raise ValueError(f"Graph output {tensor_id} is never written") + + def serialize_graph(graph: XTS3Graph) -> bytes: """Serialize to the XTS3 wire format; constants start 16-byte aligned.""" check_runtime_limits(graph) + check_dataflow(graph) tensor_data = b"".join(_serialize_tensor(tensor) for tensor in graph.tensors) parameter_data = bytearray() operation_data = bytearray() diff --git a/backends/espressif/test/test_contracts.py b/backends/espressif/test/test_contracts.py index 07b35efc99f..4d68ae301d2 100644 --- a/backends/espressif/test/test_contracts.py +++ b/backends/espressif/test/test_contracts.py @@ -34,8 +34,10 @@ EspNNQuantizer, fused_activation_kind, ) +from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 from executorch.backends.espressif.serialization.xts3 import ( # noqa: E402 build_graph, + check_dataflow, check_runtime_limits, MAX_GRAPH_IO, MAX_GRAPH_OPERATIONS, @@ -171,3 +173,54 @@ def test_runtime_limits_reject_misaligned_int32_constants(): ] with pytest.raises(ValueError, match="4-byte aligned"): check_runtime_limits(dataclasses.replace(graph, tensors=tensors)) + + +class DropoutClassifier(nn.Module): + """MobileNetV3's classifier: Linear, Hardswish, in-place Dropout, Linear.""" + + def __init__(self): + super().__init__() + self.head = nn.Sequential( + nn.Linear(16, 32), + nn.Hardswish(inplace=True), + nn.Dropout(0.2, inplace=True), + nn.Linear(32, 10), + ) + + def forward(self, x): + return self.head(x) + + +def test_dropout_between_quantized_ops_reads_the_producer(): + converted, program = quantize(DropoutClassifier(), [(1, 16)]) + graph = build_graph(program) + check_dataflow(graph) + last = graph.operations[-1] + assert last.opcode == XTS3OpCode.LINEAR + assert last.inputs[0] in graph.operations[-2].outputs + + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert report["partitions"] == 1 + x = torch.randn(1, 16) + expected = converted(x) + actual = ProgramEmulator(edge.exported_program())(x).to(torch.float32) + assert torch.allclose(actual, expected, atol=0.2) + + +def test_dataflow_rejects_reading_an_unwritten_activation(): + graph = _small_graph() + unwritten = dataclasses.replace( + graph.tensors[graph.operations[-1].inputs[0]], + tensor_id=len(graph.tensors), + ) + last = dataclasses.replace( + graph.operations[-1], + inputs=[unwritten.tensor_id, *graph.operations[-1].inputs[1:]], + ) + broken = dataclasses.replace( + graph, + tensors=[*graph.tensors, unwritten], + operations=[*graph.operations[:-1], last], + ) + with pytest.raises(ValueError, match="nothing writes"): + check_dataflow(broken) From d0c2dd73248ebcd13229b7278eb6344c56a20e1a Mon Sep 17 00:00:00 2001 From: Mwaniki Njagi Date: Tue, 27 Oct 2026 14:26:43 +0100 Subject: [PATCH 08/10] Port camera_stream to the ExecuTorch runtime Load the model as a .pte with Program/Method, planned memory in PSRAM and the ESP-NN delegate. export_model.py calibrates, runs QAT and reports top-1 of the lowered program. Default camera XCLK is 10 MHz because the OV3660 corrupts frames at 20 MHz. --- examples/espressif/camera_stream/README.md | 53 +++-- .../espressif/camera_stream/export_model.py | 213 +++++++++++++++++ examples/espressif/camera_stream/flash.sh | 2 +- .../camera_stream/main/CMakeLists.txt | 36 ++- .../camera_stream/main/Kconfig.projbuild | 17 +- .../camera_stream/main/camera_stream.cpp | 108 ++++----- .../espressif/camera_stream/main/et_model.cpp | 223 ++++++++++++++++++ .../espressif/camera_stream/main/et_model.h | 67 ++++++ examples/espressif/camera_stream/viewer.py | 7 +- 9 files changed, 631 insertions(+), 95 deletions(-) create mode 100644 examples/espressif/camera_stream/export_model.py create mode 100644 examples/espressif/camera_stream/main/et_model.cpp create mode 100644 examples/espressif/camera_stream/main/et_model.h diff --git a/examples/espressif/camera_stream/README.md b/examples/espressif/camera_stream/README.md index 1b4fb9c0b6e..8aed574f47d 100644 --- a/examples/espressif/camera_stream/README.md +++ b/examples/espressif/camera_stream/README.md @@ -1,30 +1,55 @@ # Camera inference stream (ESP32-S3) -Live ImageNet classification on an ESP32-S3 camera board with the standalone -`espnn_runtime`, streamed over USB to a terminal or a browser. +Live ImageNet classification on an ESP32-S3 camera board. The model is an +ExecuTorch program (`.pte`) run by the ExecuTorch runtime, with its int8 +graph in the ESP-NN delegate (`backends/espressif`). Results and a preview +stream over USB to a terminal or a browser. Tested with an OV3660 on the ESP32-S3-EYE / Freenove ESP32-S3-WROOM CAM pinout (selectable in menuconfig, as is the XIAO ESP32S3 Sense). ```bash -# 1. Compile a model (QAT recovers MobileNetV3-Small accuracy; plain PTQ loses most of it) -python ../aot_espnn_compiler.py --model mobilenet_v3_small --mode standalone \ - --calib-dir imagenette2-160/train --eval-dir imagenette2-160/val \ - --calib-samples 1024 --qat-epochs 3 --qat-lr 1e-4 --output models/mobilenet_v3_small_224.xtb +# 1. Quantize and lower a model. MobileNetV3-Small needs QAT; plain PTQ loses +# most of its accuracy. +python export_model.py --calib-dir imagenette2-160/train \ + --eval-dir imagenette2-160/val --qat-epochs 5 --qat-lr 1e-3 \ + --output models/mobilenet_v3_small_224.pte -# 2. Build and flash (activated ESP-IDF shell) -./flash.sh /dev/ttyACM0 +# 2. Cross-compile ExecuTorch (see backends/espressif/README.md) into +# cmake-out-esp, then build and flash from an activated ESP-IDF shell +ET_BUILD_DIR=$PWD/../../../cmake-out-esp ./flash.sh /dev/ttyACM0 # 3. Watch: terminal dashboard, or Chrome at http://127.0.0.1:8765/ -python viewer.py -python viewer.py --web +python viewer.py --port /dev/ttyACM0 +python viewer.py --port /dev/ttyACM0 --web ``` -The board captures JPEG frames from the sensor, decodes and center-crops them, -runs the model on core 1 and streams results, the model's input image and a -15 fps preview on core 0. On the ESP32-S3 with MobileNetV3-Small this gives -about 1.4 results per second with roughly 0.75 s from capture to result. +`export_model.py` prints the CPU operators the program needs and writes them +to `models/mobilenet_v3_small_224.ops.txt` for `EXECUTORCH_SELECT_OPS_LIST`. +MobileNetV3-Small lowers to one delegate with no CPU operators, so any +selection works. + +The board captures JPEG frames from the sensor, decodes and center-crops +them, runs the model on core 1, and streams results, the model's input image +and a preview of about 12 fps on core 0. + +Measured on an ESP32-S3 N16R8 at 240 MHz with MobileNetV3-Small at 224×224: + +| | | +|---|---| +| Inference | 646 ms, of which 626 ms in the ESP-NN delegate (593 ms in kernels) | +| Results | 1.3 per second, about 0.8 s from capture to result | +| Accuracy (200 Imagenette val images, 1000-way top-1) | float 134, int8 program 152 | + +The int8 program beats the float model there because QAT fine-tunes on +Imagenette's 10 classes. It is not a general ImageNet result. + +The delegate copies the weights into PSRAM and can split kernels across both +cores (`CONFIG_ESPNN_DELEGATE_DUAL_CORE`, which slows the preview). Notes from bring-up: capture JPEG rather than RGB565 (continuous RGB565 capture competes with inference for PSRAM bandwidth), keep the JPEG frame buffers in internal RAM, and do not use the camera driver's PSRAM DMA mode. +Keep XCLK at 10 MHz (the default): at 20 MHz the OV3660 started dropping +and corrupting frames after a few minutes, which showed up as a black or +frozen preview and `dec_err` counts in the results. diff --git a/examples/espressif/camera_stream/export_model.py b/examples/espressif/camera_stream/export_model.py new file mode 100644 index 00000000000..ed781ad35e4 --- /dev/null +++ b/examples/espressif/camera_stream/export_model.py @@ -0,0 +1,213 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Quantize a torchvision ImageNet classifier for the camera demo and lower it +to an ExecuTorch program (.pte) with ESP-NN delegates. + +Calibrates on an image folder, optionally fine-tunes with quantization-aware +training (distillation from the float model, no labels needed), and reports +top-1 accuracy of the lowered program as the board computes it. Also writes +.ops.txt, the CPU operators to select when building the runtime. + + python export_model.py --calib-dir imagenette2-160/train \\ + --eval-dir imagenette2-160/val --qat-epochs 5 --qat-lr 1e-3 \\ + --output models/mobilenet_v3_small_224.pte + +MobileNetV3-Small needs QAT; plain PTQ loses most of its accuracy. +""" + +from __future__ import annotations + +import argparse +import copy +import random +from pathlib import Path + +import torch +import torch.nn.functional as F +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from executorch.backends.espressif.reference import ProgramEmulator +from torchao.quantization.pt2e import ( + disable_observer, + move_exported_model_to_eval, + move_exported_model_to_train, +) +from torchao.quantization.pt2e.quantize_pt2e import ( + convert_pt2e, + prepare_pt2e, + prepare_qat_pt2e, +) + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} +# Imagenette folder names and their ImageNet-1k class indices. +IMAGENETTE_TO_IMAGENET = { + "n01440764": 0, + "n02102040": 217, + "n02979186": 482, + "n03000684": 491, + "n03028079": 497, + "n03394916": 566, + "n03417042": 569, + "n03425413": 571, + "n03445777": 574, + "n03888257": 701, +} +MEAN = (0.485, 0.456, 0.406) +STD = (0.229, 0.224, 0.225) + + +def load_images(directory: Path, count: int, size: int, seed: int): + from PIL import Image + from torchvision import transforms + + transform = transforms.Compose( + [ + transforms.Resize(size * 256 // 224), + transforms.CenterCrop(size), + transforms.ToTensor(), + transforms.Normalize(MEAN, STD), + ] + ) + files = sorted( + p for p in directory.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES + ) + if not files: + raise SystemExit(f"no images found under {directory}") + random.Random(seed).shuffle(files) + images, labels = [], [] + for path in files[:count]: + images.append(transform(Image.open(path).convert("RGB"))) + labels.append(IMAGENETTE_TO_IMAGENET.get(path.parent.name, -1)) + return torch.stack(images), labels + + +def qat_prepare(model, data, args): + batch = min(args.qat_batch, len(data)) + exported = torch.export.export( + copy.deepcopy(model), + (data[:2],), + dynamic_shapes=({0: torch.export.Dim("batch", min=2, max=1024)},), + strict=True, + ).module() + prepared = prepare_qat_pt2e(exported, EspNNQuantizer(is_qat=True)) + parameters = [p for p in prepared.parameters() if p.is_floating_point()] + for parameter in parameters: + parameter.requires_grad_(True) + steps = max(1, round(args.qat_epochs * max(1, len(data) // batch))) + optimizer = torch.optim.SGD(parameters, lr=args.qat_lr, momentum=0.9) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, steps) + generator = torch.Generator().manual_seed(args.seed) + move_exported_model_to_train(prepared) + with torch.no_grad(): + for start in range(0, min(len(data), 16 * batch), batch): + prepared(data[start : start + batch]) + with torch.enable_grad(): + for step in range(steps): + if step == int(steps * 0.6): + prepared.apply(disable_observer) + inputs = data[torch.randint(0, len(data), (batch,), generator=generator)] + if bool(torch.rand(1, generator=generator) < 0.5): + inputs = inputs.flip(3) + with torch.no_grad(): + teacher = F.softmax(model(inputs), dim=1) + loss = F.kl_div( + F.log_softmax(prepared(inputs), dim=1), teacher, reduction="batchmean" + ) + optimizer.zero_grad() + loss.backward() + optimizer.step() + scheduler.step() + if step % max(1, steps // 10) == 0 or step + 1 == steps: + print(f"QAT step {step + 1}/{steps} loss {loss.item():.4f}") + move_exported_model_to_eval(prepared) + for parameter in parameters: + parameter.requires_grad_(False) + return prepared + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="mobilenet_v3_small") + parser.add_argument("--size", type=int, default=224) + parser.add_argument("--calib-dir", type=Path, required=True) + parser.add_argument("--calib-samples", type=int, default=1024) + parser.add_argument("--eval-dir", type=Path) + parser.add_argument("--eval-samples", type=int, default=100) + parser.add_argument("--qat-epochs", type=float, default=0.0) + parser.add_argument("--qat-lr", type=float, default=1e-3) + parser.add_argument("--qat-batch", type=int, default=16) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--output", type=Path, default=Path("models/mobilenet_v3_small_224.pte") + ) + args = parser.parse_args() + + from torchvision import models + + model = getattr(models, args.model)(weights="DEFAULT").eval() + calibration, _ = load_images( + args.calib_dir, args.calib_samples, args.size, args.seed + ) + example = (calibration[:1],) + + if args.qat_epochs > 0: + prepared = qat_prepare(model, calibration, args) + else: + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), + EspNNQuantizer(), + ) + with torch.no_grad(): + for image in calibration: + prepared(image.unsqueeze(0)) + quantized = convert_pt2e(prepared) + + edge, report = to_espnn_edge(torch.export.export(quantized, example, strict=True)) + print( + f"{report['partitions']} ESP-NN partition(s), " + f"{report['delegated_ops']} delegated operators, " + f"{len(report['cpu_ops'])} on the CPU" + ) + + if args.eval_dir is not None: + images, labels = load_images( + args.eval_dir, args.eval_samples, args.size, args.seed + 1 + ) + emulator = ProgramEmulator(edge.exported_program()) + float_correct = device_correct = agree = 0 + with torch.no_grad(): + for image, label in zip(images, labels): + image = image.unsqueeze(0) + float_top1 = int(model(image).argmax()) + device_top1 = int(emulator(image).to(torch.float32).argmax()) + float_correct += int(float_top1 == label) + device_correct += int(device_top1 == label) + agree += int(float_top1 == device_top1) + print( + f"top-1 on {len(labels)} images: float {float_correct}, " + f"int8 program {device_correct}, agreement {agree}" + ) + + program = edge.to_executorch() + plan = program.executorch_program.execution_plan[0] + operators = sorted( + f"{op.name}.{op.overload}" + for op in plan.operators + if not op.name.startswith("quantized_decomposed::") + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(program.buffer) + ops_path = args.output.with_suffix(".ops.txt") + ops_path.write_text(",".join(operators) + "\n") + print(f"wrote {args.output} ({len(program.buffer)} bytes) and {ops_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/camera_stream/flash.sh b/examples/espressif/camera_stream/flash.sh index 7f7188dc318..3fa14895050 100755 --- a/examples/espressif/camera_stream/flash.sh +++ b/examples/espressif/camera_stream/flash.sh @@ -13,7 +13,7 @@ set -euo pipefail PORT=${1:-${ESPPORT:?usage: flash.sh PORT (or set ESPPORT)}} cd "$(dirname "$0")" mkdir -p build -idf.py build +idf.py ${ET_BUILD_DIR:+-DET_BUILD_DIR_PATH="$ET_BUILD_DIR"} build touch build/.viewer_pause trap 'rm -f build/.viewer_pause' EXIT sleep 1 diff --git a/examples/espressif/camera_stream/main/CMakeLists.txt b/examples/espressif/camera_stream/main/CMakeLists.txt index 67d0802ead0..4a8dfd61e06 100644 --- a/examples/espressif/camera_stream/main/CMakeLists.txt +++ b/examples/espressif/camera_stream/main/CMakeLists.txt @@ -6,11 +6,19 @@ # # SPDX-License-Identifier: BSD-3-Clause +get_filename_component( + camera_et_root "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE +) + +# The ExecuTorch platform layer is shared with the executor runner example. +# WHOLE_ARCHIVE keeps it over the runtime's weak defaults. idf_component_register( SRCS "main.cpp" "camera_stream.cpp" + "et_model.cpp" "model_data.S" + "${camera_et_root}/examples/espressif/executor_runner/esp_pal.cpp" INCLUDE_DIRS "." REQUIRES @@ -20,7 +28,31 @@ idf_component_register( esp_timer esp_rom esp_driver_usb_serial_jtag + WHOLE_ARCHIVE +) + +set(ET_BUILD_DIR_PATH + "${camera_et_root}/cmake-out-esp" + CACHE PATH "ExecuTorch cross-compiled for this target and installed" ) +# GLOBAL so that the project-level link step and espnn_runtime see the imported +# targets. +set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL TRUE) +find_package( + executorch REQUIRED HINTS "${ET_BUILD_DIR_PATH}/lib/cmake/ExecuTorch" +) +target_include_directories( + ${COMPONENT_LIB} PRIVATE "${camera_et_root}/.." + "${camera_et_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + ${COMPONENT_LIB} PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +set(camera_et_libs executorch executorch_selected_kernels) +if(TARGET quantized_ops_lib) + list(APPEND camera_et_libs quantized_ops_lib) +endif() +target_link_libraries(${COMPONENT_LIB} PRIVATE ${camera_et_libs}) idf_build_get_property(project_dir PROJECT_DIR) if(IS_ABSOLUTE "${CONFIG_CAMERA_STREAM_MODEL}") @@ -31,8 +63,8 @@ endif() if(NOT EXISTS "${model}") message( FATAL_ERROR - "Camera model ${model} not found. Compile one with " - "examples/espressif/aot_espnn_compiler.py --mode standalone and set " + "Camera model ${model} not found. Export one with " + "examples/espressif/camera_stream/export_model.py and set " "CONFIG_CAMERA_STREAM_MODEL." ) endif() diff --git a/examples/espressif/camera_stream/main/Kconfig.projbuild b/examples/espressif/camera_stream/main/Kconfig.projbuild index 7164391251d..6a5cf38928f 100644 --- a/examples/espressif/camera_stream/main/Kconfig.projbuild +++ b/examples/espressif/camera_stream/main/Kconfig.projbuild @@ -9,8 +9,8 @@ menu "Camera inference stream" config CAMERA_STREAM_MODEL - string "Model to embed (XTS3 graph or XTB1 bundle; absolute or relative to this project)" - default "models/mobilenet_v3_small_224.xtb" + string "ExecuTorch program (.pte) to embed; absolute or relative to this project" + default "models/mobilenet_v3_small_224.pte" choice CAMERA_STREAM_PINOUT @@ -38,17 +38,14 @@ menu "Camera inference stream" default 12 depends on CAMERA_STREAM_SENSOR_JPEG - config CAMERA_STREAM_DUAL_CORE - bool "Split ESP-NN kernels across both cores" - default n - help - Faster inference, but the ESP-NN worker on core 0 competes with - the JPEG preview task, so the preview frame rate drops. - config CAMERA_STREAM_XCLK_MHZ int "Camera XCLK frequency in MHz" range 6 24 - default 20 + default 10 + help + At 20 MHz an OV3660 on the ESP32-S3 CAM boards drops and corrupts + frames after a while (NO-SOI, frame timeouts, dark frames). + 10 MHz is stable and still gives a 12 fps preview. config CAMERA_STREAM_PREVIEW bool "Stream JPEG preview frames alongside the results" diff --git a/examples/espressif/camera_stream/main/camera_stream.cpp b/examples/espressif/camera_stream/main/camera_stream.cpp index 7aa9640e219..4360e16c6ae 100644 --- a/examples/espressif/camera_stream/main/camera_stream.cpp +++ b/examples/espressif/camera_stream/main/camera_stream.cpp @@ -31,7 +31,7 @@ #include #include -#include +#include extern "C" { extern const uint8_t camera_model_start[]; @@ -42,11 +42,6 @@ namespace executorch { namespace examples { namespace espressif { -using ::executorch::backends::espressif::Int8Interpreter; -using ::executorch::backends::espressif::Int8RuntimeOptions; -using ::executorch::backends::espressif::Int8TensorInfo; -using ::executorch::backends::espressif::read_le_u32; - namespace { constexpr int TOP_K = 5; @@ -78,6 +73,11 @@ constexpr bool PREVIEW = false; constexpr int PREVIEW_QUALITY = 50; constexpr int PREVIEW_MAX_FPS = 1; #endif +#if CONFIG_ESPNN_DELEGATE_DUAL_CORE +constexpr bool DUAL_CORE = true; +#else +constexpr bool DUAL_CORE = false; +#endif #if CONFIG_CAMERA_STREAM_SENSOR_JPEG constexpr bool SENSOR_JPEG = true; constexpr int SENSOR_JPEG_QUALITY = CONFIG_CAMERA_STREAM_SENSOR_JPEG_QUALITY; @@ -143,6 +143,7 @@ struct Result { int64_t decode_us; int64_t preprocess_us; int64_t invoke_us; + int64_t delegate_us; int64_t cycle_us; int64_t latency_us; uint32_t dropped_results; @@ -153,7 +154,7 @@ struct Result { struct Shared { SemaphoreHandle_t state_lock; QueueHandle_t results; - Int8Interpreter* interpreter; + EtModel* model; int32_t input_width; int32_t input_height; bool decoder_swaps_red_blue; @@ -540,13 +541,10 @@ class Resampler { uint16_t* y_weight_ = nullptr; }; -bool top_k( - const Int8Interpreter& interpreter, - float* scratch, - size_t count, - Prediction* top) { - if (!interpreter.dequantize_output(0, scratch, count)) +bool top_k(const float* logits, float* scratch, size_t count, Prediction* top) { + if (logits == nullptr) return false; + std::memcpy(scratch, logits, sizeof(float) * count); float maximum = scratch[0]; for (size_t i = 1; i < count; ++i) { if (scratch[i] > maximum) @@ -577,15 +575,13 @@ bool top_k( } void inference_task(void*) { - Int8Interpreter& interpreter = *g_shared.interpreter; - Int8TensorInfo output_info{}; - interpreter.output_info(0, output_info); + EtModel& model = *g_shared.model; + const size_t classes = model.info().classes; const int32_t width = g_shared.input_width; const int32_t height = g_shared.input_height; const size_t rgb_bytes = static_cast(width) * height * 3; uint8_t* rgb = static_cast(psram_alloc(rgb_bytes)); - float* scratch = - static_cast(std::malloc(sizeof(float) * output_info.bytes)); + float* scratch = static_cast(std::malloc(sizeof(float) * classes)); uint8_t* decoded = nullptr; size_t decoded_size = 0; Resampler resampler; @@ -646,19 +642,17 @@ void inference_task(void*) { resampler.run_rgb565_be(frame->buf, rgb); esp_camera_fb_return(frame); } - interpreter.set_input_rgb888( - 0, rgb, width, height, IMAGENET_MEAN, IMAGENET_STD); + model.set_input_rgb888(rgb, IMAGENET_MEAN, IMAGENET_STD); const int64_t preprocessed = esp_timer_get_time(); - if (!interpreter.invoke()) { - std::printf("#ERR invoke failed\n"); + if (!model.run()) continue; - } Result result{}; result.seq = ++seq; result.decode_us = decode_us; result.preprocess_us = preprocessed - started; - result.invoke_us = interpreter.last_invoke_us(); - top_k(interpreter, scratch, output_info.bytes, result.top); + result.invoke_us = model.last_run_us(); + result.delegate_us = model.last_delegate_us(); + top_k(model.logits(), scratch, classes, result.top); const int64_t now = esp_timer_get_time(); result.cycle_us = now - previous; result.latency_us = now - captured; @@ -688,12 +682,14 @@ void send_result(UsbLink& link, const Result& result) { int n = std::snprintf( line, sizeof(line), - "#RES seq=%u dec_us=%lld pre_us=%lld infer_us=%lld cycle_us=%lld latency_us=%lld " + "#RES seq=%u dec_us=%lld pre_us=%lld infer_us=%lld delegate_us=%lld cycle_us=%lld " + "latency_us=%lld " "dropped=%u dec_err=%u usb_fail=%u top=", static_cast(result.seq), static_cast(result.decode_us), static_cast(result.preprocess_us), static_cast(result.invoke_us), + static_cast(result.delegate_us), static_cast(result.cycle_us), static_cast(result.latency_us), static_cast(result.dropped_results), @@ -918,45 +914,22 @@ void run_camera_stream() { if (!start_camera()) return; - Int8RuntimeOptions options{}; - options.copy_constants_to_psram = true; -#if CONFIG_CAMERA_STREAM_DUAL_CORE - options.dual_core = true; -#endif - Int8Interpreter* interpreter = new (std::nothrow) Int8Interpreter(); - const uint8_t* model = camera_model_start; - size_t model_size = reinterpret_cast(camera_model_end) - + EtModel* model = new (std::nothrow) EtModel(); + const size_t model_size = reinterpret_cast(camera_model_end) - reinterpret_cast(camera_model_start); - if (model_size >= 12 + 48 && std::memcmp(model, "XTB1", 4) == 0) { - const uint32_t offset = read_le_u32(model + 12 + 32); - const uint32_t length = read_le_u32(model + 12 + 36); - if (offset > model_size || length > model_size - offset) { - std::printf("#ERR model bundle is truncated\n"); - return; - } - model += offset; - model_size = length; - } - if (interpreter == nullptr || - !interpreter->load(model, model_size, options)) { + if (model == nullptr || !model->load(camera_model_start, model_size)) { std::printf("#ERR model load failed\n"); return; } - Int8TensorInfo input{}; - Int8TensorInfo output{}; - if (interpreter->input_count() != 1 || !interpreter->input_info(0, input) || - !interpreter->output_info(0, output) || input.channels != 3) { - std::printf("#ERR model must have one 3-channel image input\n"); - return; - } - g_shared.interpreter = interpreter; - g_shared.input_width = input.width; - g_shared.input_height = input.height; + const EtModelInfo& info = model->info(); + g_shared.model = model; + g_shared.input_width = info.input_width; + g_shared.input_height = info.input_height; g_shared.state_lock = xSemaphoreCreateMutex(); g_shared.results = xQueueCreate(RESULT_QUEUE_LENGTH, sizeof(Result)); g_shared.published_bgr = PREVIEW - ? static_cast( - psram_alloc(static_cast(input.width) * input.height * 3)) + ? static_cast(psram_alloc( + static_cast(info.input_width) * info.input_height * 3)) : nullptr; if (g_shared.state_lock == nullptr || g_shared.results == nullptr || (PREVIEW && g_shared.published_bgr == nullptr)) { @@ -964,18 +937,21 @@ void run_camera_stream() { return; } std::printf( - "#CAM model_input=%ldx%ld classes=%u dual_core=%d preview=%d preview_fps=%d " - "sensor=%s sensor_quality=%d xclk_mhz=%d constants_in_psram=%d\n", - static_cast(input.width), - static_cast(input.height), - static_cast(output.bytes), - options.dual_core ? 1 : 0, + "#CAM model_input=%ldx%ld classes=%u runtime=executorch delegates=%u " + "delegate_arena=%u planned=%u dual_core=%d preview=%d preview_fps=%d " + "sensor=%s sensor_quality=%d xclk_mhz=%d\n", + static_cast(info.input_width), + static_cast(info.input_height), + static_cast(info.classes), + static_cast(info.delegates), + static_cast(info.delegate_arena_bytes), + static_cast(info.planned_bytes), + DUAL_CORE ? 1 : 0, PREVIEW ? 1 : 0, PREVIEW_MAX_FPS, SENSOR_JPEG ? "jpeg" : "rgb565", SENSOR_JPEG_QUALITY, - CONFIG_CAMERA_STREAM_XCLK_MHZ, - interpreter->constants_copied() ? 1 : 0); + CONFIG_CAMERA_STREAM_XCLK_MHZ); std::printf( "Internal RAM free %u (largest %u), PSRAM free %u\n", static_cast( diff --git a/examples/espressif/camera_stream/main/et_model.cpp b/examples/espressif/camera_stream/main/et_model.cpp new file mode 100644 index 00000000000..c96b38b40f3 --- /dev/null +++ b/examples/espressif/camera_stream/main/et_model.cpp @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace executorch { +namespace examples { +namespace espressif { + +using ::executorch::aten::ScalarType; +using ::executorch::aten::Tensor; +using ::executorch::extension::BufferDataLoader; +using ::executorch::runtime::Error; +using ::executorch::runtime::EValue; +using ::executorch::runtime::HierarchicalAllocator; +using ::executorch::runtime::MemoryAllocator; +using ::executorch::runtime::MemoryManager; +using ::executorch::runtime::Method; +using ::executorch::runtime::MethodMeta; +using ::executorch::runtime::Program; +using ::executorch::runtime::Result; +using ::executorch::runtime::Span; +using ::executorch::runtime::etensor::TensorImpl; + +namespace { + +constexpr size_t METHOD_POOL_BYTES = 256 * 1024; +constexpr size_t TEMP_POOL_BYTES = 128 * 1024; + +uint8_t* psram_alloc(size_t bytes) { + void* pointer = + heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (pointer == nullptr) + pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_8BIT); + return static_cast(pointer); +} + +bool float_tensor(Result<::executorch::runtime::TensorInfo> meta, size_t rank) { + return meta.ok() && meta->scalar_type() == ScalarType::Float && + meta->sizes().size() == rank; +} + +} // namespace + +bool EtModel::load(const uint8_t* data, size_t size) { + ::executorch::runtime::runtime_init(); + auto* loader = new (std::nothrow) BufferDataLoader(data, size); + if (loader == nullptr) + return false; + Result program = Program::load(loader); + if (!program.ok()) { + std::printf("#ERR Program::load failed: 0x%x\n", unsigned(program.error())); + return false; + } + auto* owned_program = new (std::nothrow) Program(std::move(*program)); + if (owned_program == nullptr) + return false; + Result meta = owned_program->method_meta("forward"); + if (!meta.ok() || meta->num_inputs() != 1 || meta->num_outputs() != 1 || + !float_tensor(meta->input_tensor_meta(0), 4) || + !float_tensor(meta->output_tensor_meta(0), 2)) { + std::printf( + "#ERR model must map one float [1,3,H,W] image to float [1,classes]\n"); + return false; + } + auto input_meta = meta->input_tensor_meta(0); + auto output_meta = meta->output_tensor_meta(0); + if (input_meta->sizes()[0] != 1 || input_meta->sizes()[1] != 3 || + output_meta->sizes()[0] != 1) { + std::printf("#ERR model input must be [1,3,H,W]\n"); + return false; + } + + uint8_t* method_pool = psram_alloc(METHOD_POOL_BYTES); + uint8_t* temp_pool = psram_alloc(TEMP_POOL_BYTES); + const size_t planned_count = meta->num_memory_planned_buffers(); + auto* planned_spans = new (std::nothrow) Span[planned_count + 1]; + if (method_pool == nullptr || temp_pool == nullptr || + planned_spans == nullptr) { + std::printf("#ERR runtime pools could not be allocated\n"); + return false; + } + size_t planned_bytes = 0; + for (size_t i = 0; i < planned_count; ++i) { + Result bytes = meta->memory_planned_buffer_size(i); + if (!bytes.ok() || *bytes < 0) + return false; + uint8_t* buffer = psram_alloc(static_cast(*bytes)); + if (buffer == nullptr) { + std::printf( + "#ERR planned buffer %u (%lld bytes) could not be allocated\n", + unsigned(i), + static_cast(*bytes)); + return false; + } + planned_spans[i] = {buffer, static_cast(*bytes)}; + planned_bytes += static_cast(*bytes); + } + + auto* method_allocator = + new (std::nothrow) MemoryAllocator(METHOD_POOL_BYTES, method_pool); + auto* temp_allocator = + new (std::nothrow) MemoryAllocator(TEMP_POOL_BYTES, temp_pool); + auto* planned = + new (std::nothrow) HierarchicalAllocator({planned_spans, planned_count}); + if (method_allocator == nullptr || temp_allocator == nullptr || + planned == nullptr) + return false; + auto* memory = new (std::nothrow) + MemoryManager(method_allocator, planned, temp_allocator); + if (memory == nullptr) + return false; + + const auto before = ::executorch::backends::espressif::delegate_stats(); + Result method = owned_program->load_method("forward", memory); + if (!method.ok()) { + std::printf("#ERR load_method failed: 0x%x\n", unsigned(method.error())); + return false; + } + const auto after = ::executorch::backends::espressif::delegate_stats(); + method_ = new (std::nothrow) Method(std::move(*method)); + if (method_ == nullptr) + return false; + + info_.input_height = input_meta->sizes()[2]; + info_.input_width = input_meta->sizes()[3]; + info_.classes = static_cast(output_meta->sizes()[1]); + info_.planned_bytes = planned_bytes; + info_.delegates = after.delegates - before.delegates; + info_.delegate_arena_bytes = after.arena_bytes - before.arena_bytes; + + const size_t pixels = + static_cast(info_.input_width) * info_.input_height; + input_ = reinterpret_cast(psram_alloc(sizeof(float) * pixels * 3)); + if (input_ == nullptr) + return false; + sizes_[0] = 1; + sizes_[1] = 3; + sizes_[2] = info_.input_height; + sizes_[3] = info_.input_width; + for (int i = 0; i < 4; ++i) + dim_order_[i] = static_cast(i); + strides_[3] = 1; + strides_[2] = info_.input_width; + strides_[1] = static_cast(pixels); + strides_[0] = static_cast(pixels * 3); + input_impl_ = new (std::nothrow) + TensorImpl(ScalarType::Float, 4, sizes_, input_, dim_order_, strides_); + return input_impl_ != nullptr; +} + +bool EtModel::set_input_rgb888( + const uint8_t* rgb, + const float mean[3], + const float stddev[3]) { + if (input_ == nullptr || rgb == nullptr) + return false; + float lut[3][256]; + for (int c = 0; c < 3; ++c) { + for (int v = 0; v < 256; ++v) + lut[c][v] = (static_cast(v) / 255.0f - mean[c]) / stddev[c]; + } + const size_t pixels = + static_cast(info_.input_width) * info_.input_height; + float* red = input_; + float* green = input_ + pixels; + float* blue = input_ + 2 * pixels; + for (size_t i = 0; i < pixels; ++i) { + red[i] = lut[0][rgb[i * 3 + 0]]; + green[i] = lut[1][rgb[i * 3 + 1]]; + blue[i] = lut[2][rgb[i * 3 + 2]]; + } + return true; +} + +bool EtModel::run() { + if (method_ == nullptr) + return false; + const int64_t started = esp_timer_get_time(); + const auto before = ::executorch::backends::espressif::delegate_stats(); + Error error = method_->set_input(EValue(Tensor(input_impl_)), 0); + if (error == Error::Ok) + error = method_->execute(); + const auto after = ::executorch::backends::espressif::delegate_stats(); + last_run_us_ = esp_timer_get_time() - started; + last_delegate_us_ = after.total_us - before.total_us; + if (error != Error::Ok) { + std::printf("#ERR execute failed: 0x%x\n", unsigned(error)); + return false; + } + const EValue& output = method_->get_output(0); + return output.isTensor() && + output.toTensor().numel() == static_cast(info_.classes); +} + +const float* EtModel::logits() const { + return method_ != nullptr + ? method_->get_output(0).toTensor().const_data_ptr() + : nullptr; +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/et_model.h b/examples/espressif/camera_stream/main/et_model.h new file mode 100644 index 00000000000..21aaa1c7bdf --- /dev/null +++ b/examples/espressif/camera_stream/main/et_model.h @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace examples { +namespace espressif { + +struct EtModelInfo { + int32_t input_width; + int32_t input_height; + size_t classes; + size_t planned_bytes; + uint32_t delegates; + size_t delegate_arena_bytes; +}; + +class EtModel { + public: + bool load(const uint8_t* program, size_t size); + const EtModelInfo& info() const { + return info_; + } + bool set_input_rgb888( + const uint8_t* rgb, + const float mean[3], + const float stddev[3]); + bool run(); + int64_t last_run_us() const { + return last_run_us_; + } + int64_t last_delegate_us() const { + return last_delegate_us_; + } + const float* logits() const; + + private: + using TensorImpl = ::executorch::runtime::etensor::TensorImpl; + + ::executorch::runtime::Method* method_ = nullptr; + float* input_ = nullptr; + TensorImpl* input_impl_ = nullptr; + TensorImpl::SizesType sizes_[4] = {}; + TensorImpl::DimOrderType dim_order_[4] = {}; + TensorImpl::StridesType strides_[4] = {}; + EtModelInfo info_{}; + int64_t last_run_us_ = 0; + int64_t last_delegate_us_ = 0; +}; + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/viewer.py b/examples/espressif/camera_stream/viewer.py index e3d49ed742f..1ae3a0b45aa 100644 --- a/examples/espressif/camera_stream/viewer.py +++ b/examples/espressif/camera_stream/viewer.py @@ -12,7 +12,7 @@ The board prints text lines on its USB-Serial-JTAG port: #CAM ... stream configuration #CROP x= y= side= centre crop fed to the model, in camera pixels - #RES seq= pre_us= infer_us= cycle_us= top=idx:prob,... + #RES seq= pre_us= infer_us= delegate_us= cycle_us= top=idx:prob,... #JPG kind=cam|input seq= res= w= h= enc_us= len= crc= b64=... Terminal dashboard (stdlib only; --preview draws the frame with Pillow): @@ -174,6 +174,7 @@ def handle_line(self, raw: bytes) -> None: "seq": int(fields.get("seq", 0)), "pre_ms": int(fields.get("pre_us", 0)) / 1000.0, "infer_ms": int(fields.get("infer_us", 0)) / 1000.0, + "delegate_ms": int(fields.get("delegate_us", 0)) / 1000.0, "cycle_ms": int(fields.get("cycle_us", 0)) / 1000.0, "latency_ms": int(fields.get("latency_us", 0)) / 1000.0, "decode_ms": int(fields.get("dec_us", 0)) / 1000.0, @@ -402,6 +403,7 @@ def reader_thread( # noqa: C901
-
inferences / s
-
inference ms
+
-
in ESP-NN ms
-
preprocess ms
-
preview fps
USB KB/s
@@ -466,13 +468,14 @@ def reader_thread( # noqa: C901 $("m-pfps").textContent = fmt(st.preview_fps, 1); $("m-link").textContent = fmt(st.link_kbps, 0); const cfg = st.config || {}; - $("model").textContent = cfg.model_input ? `input ${cfg.model_input}, ${cfg.classes} classes, dual core ${cfg.dual_core === "1" ? "on" : "off"}` : ""; + $("model").textContent = cfg.model_input ? `${cfg.runtime || "standalone"} runtime, ${cfg.delegates || "?"} ESP-NN delegate(s), input ${cfg.model_input}, ${cfg.classes} classes, dual core ${cfg.dual_core === "1" ? "on" : "off"}` : ""; const live = st.result_age_ms !== null && st.result_age_ms < 5000; $("dot").style.background = live ? "var(--good)" : "var(--warn)"; $("status-text").textContent = st.link && st.link !== "connected" ? st.link : (live ? "streaming" : (r ? "stalled" : "waiting for board")); if (live && camStale) { camStale = false; $("cam").src = "stream.mjpg?t=" + Date.now(); } if (r) { $("m-infer").textContent = fmt(r.infer_ms, 0); + $("m-deleg").textContent = r.delegate_ms ? fmt(r.delegate_ms, 0) : "-"; $("m-pre").textContent = fmt(r.pre_ms, 0); $("m-lat").textContent = fmt(r.latency_ms, 0); if (r.seq !== lastSeq) { From d64af3b120a7d2d7918d8ef94ce4ba16211bb8fb Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Tue, 27 Oct 2026 13:45:35 -0700 Subject: [PATCH 09/10] Fix Espressif test isolation and camera input allocation Scope no-grad mode to individual Espressif tests so collection does not disable autograd for unrelated suites. Preserve optional ESP-NN hooks and correct the camera allocator's pointer casts to clear cppcheck findings. Let camera exports use the application's input buffer, saving a duplicate 588 KiB allocation for 224x224 RGB input and avoiding the input copy. Document that existing models need re-exporting. Validated with 63 host and regression tests, changed-file lint, and a native input-memory smoke test. macOS and hardware confirmation are still pending. Authored with assistance from OpenAI Codex. --- .../espressif/espnn_runtime/src/espnn_backend.cpp | 4 ++-- backends/espressif/test/test_cpu_fallback.py | 5 ++++- backends/espressif/test/test_int8_lowering.py | 6 +++++- backends/espressif/test/test_int8_ops.py | 5 ++++- examples/espressif/camera_stream/README.md | 4 ++++ examples/espressif/camera_stream/export_model.py | 8 +++++++- examples/espressif/camera_stream/main/et_model.cpp | 13 +++++++------ 7 files changed, 33 insertions(+), 12 deletions(-) diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp index 1be6cf89a5c..06b21783611 100644 --- a/backends/espressif/espnn_runtime/src/espnn_backend.cpp +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -278,7 +278,7 @@ bool g_registered = executorch::runtime::register_backend( } // namespace -bool register_espnn_backend() { +[[maybe_unused]] bool register_espnn_backend() { if (!g_registered) { g_registered = executorch::runtime::register_backend( Backend{kEspNNBackendId, &g_backend}) == Error::Ok; @@ -290,7 +290,7 @@ DelegateStats delegate_stats() { return g_stats; } -void reset_delegate_stats() { +[[maybe_unused]] void reset_delegate_stats() { g_stats = {}; } diff --git a/backends/espressif/test/test_cpu_fallback.py b/backends/espressif/test/test_cpu_fallback.py index 9c8746a1fb6..a43d186cb7c 100644 --- a/backends/espressif/test/test_cpu_fallback.py +++ b/backends/espressif/test/test_cpu_fallback.py @@ -29,7 +29,10 @@ from executorch.exir.lowered_backend_module import LoweredBackendModule -torch.set_grad_enabled(False) +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield def sqnr_db(reference: torch.Tensor, actual: torch.Tensor) -> float: diff --git a/backends/espressif/test/test_int8_lowering.py b/backends/espressif/test/test_int8_lowering.py index 8522d711233..c35a17f7010 100644 --- a/backends/espressif/test/test_int8_lowering.py +++ b/backends/espressif/test/test_int8_lowering.py @@ -39,7 +39,11 @@ from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e -torch.set_grad_enabled(False) +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + INT32_MIN = -(1 << 31) INT32_MAX = (1 << 31) - 1 diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py index 9429e4d2ba5..ee33d41fd23 100644 --- a/backends/espressif/test/test_int8_ops.py +++ b/backends/espressif/test/test_int8_ops.py @@ -37,7 +37,10 @@ from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e -torch.set_grad_enabled(False) +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield def quantize(model: nn.Module, shapes, samples: int = 24, seed: int = 0): diff --git a/examples/espressif/camera_stream/README.md b/examples/espressif/camera_stream/README.md index 8aed574f47d..9c995c386bd 100644 --- a/examples/espressif/camera_stream/README.md +++ b/examples/espressif/camera_stream/README.md @@ -29,6 +29,10 @@ to `models/mobilenet_v3_small_224.ops.txt` for `EXECUTORCH_SELECT_OPS_LIST`. MobileNetV3-Small lowers to one delegate with no CPU operators, so any selection works. +The exporter leaves input allocation to the camera application. The runtime +shares the normalized image buffer, avoiding a second float input buffer and +a copy on each inference. Re-export existing models to use this memory plan. + The board captures JPEG frames from the sensor, decodes and center-crops them, runs the model on core 1, and streams results, the model's input image and a preview of about 12 fps on core 0. diff --git a/examples/espressif/camera_stream/export_model.py b/examples/espressif/camera_stream/export_model.py index ed781ad35e4..e9bd25f0951 100644 --- a/examples/espressif/camera_stream/export_model.py +++ b/examples/espressif/camera_stream/export_model.py @@ -33,6 +33,8 @@ import torch.nn.functional as F from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge from executorch.backends.espressif.reference import ProgramEmulator +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass from torchao.quantization.pt2e import ( disable_observer, move_exported_model_to_eval, @@ -194,7 +196,11 @@ def main() -> int: f"int8 program {device_correct}, agreement {agree}" ) - program = edge.to_executorch() + program = edge.to_executorch( + ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass(alloc_graph_input=False) + ) + ) plan = program.executorch_program.execution_plan[0] operators = sorted( f"{op.name}.{op.overload}" diff --git a/examples/espressif/camera_stream/main/et_model.cpp b/examples/espressif/camera_stream/main/et_model.cpp index c96b38b40f3..650aad388fd 100644 --- a/examples/espressif/camera_stream/main/et_model.cpp +++ b/examples/espressif/camera_stream/main/et_model.cpp @@ -46,12 +46,12 @@ namespace { constexpr size_t METHOD_POOL_BYTES = 256 * 1024; constexpr size_t TEMP_POOL_BYTES = 128 * 1024; -uint8_t* psram_alloc(size_t bytes) { +void* psram_alloc(size_t bytes) { void* pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); if (pointer == nullptr) pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_8BIT); - return static_cast(pointer); + return pointer; } bool float_tensor(Result<::executorch::runtime::TensorInfo> meta, size_t rank) { @@ -90,8 +90,8 @@ bool EtModel::load(const uint8_t* data, size_t size) { return false; } - uint8_t* method_pool = psram_alloc(METHOD_POOL_BYTES); - uint8_t* temp_pool = psram_alloc(TEMP_POOL_BYTES); + auto* method_pool = static_cast(psram_alloc(METHOD_POOL_BYTES)); + auto* temp_pool = static_cast(psram_alloc(TEMP_POOL_BYTES)); const size_t planned_count = meta->num_memory_planned_buffers(); auto* planned_spans = new (std::nothrow) Span[planned_count + 1]; if (method_pool == nullptr || temp_pool == nullptr || @@ -104,7 +104,8 @@ bool EtModel::load(const uint8_t* data, size_t size) { Result bytes = meta->memory_planned_buffer_size(i); if (!bytes.ok() || *bytes < 0) return false; - uint8_t* buffer = psram_alloc(static_cast(*bytes)); + auto* buffer = + static_cast(psram_alloc(static_cast(*bytes))); if (buffer == nullptr) { std::printf( "#ERR planned buffer %u (%lld bytes) could not be allocated\n", @@ -150,7 +151,7 @@ bool EtModel::load(const uint8_t* data, size_t size) { const size_t pixels = static_cast(info_.input_width) * info_.input_height; - input_ = reinterpret_cast(psram_alloc(sizeof(float) * pixels * 3)); + input_ = static_cast(psram_alloc(sizeof(float) * pixels * 3)); if (input_ == nullptr) return false; sizes_[0] = 1; From b3ae9746bb981b7ed0c2c006dd56d7d20c296f4f Mon Sep 17 00:00:00 2001 From: RJ Ascani Date: Tue, 27 Oct 2026 13:49:21 -0700 Subject: [PATCH 10/10] Document Espressif backend prototype status Authored with assistance from OpenAI Codex. --- backends/espressif/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backends/espressif/README.md b/backends/espressif/README.md index 3530f35345e..89188618fc6 100644 --- a/backends/espressif/README.md +++ b/backends/espressif/README.md @@ -1,5 +1,7 @@ # ESP-NN backend for Espressif chips +This backend is a **prototype under active development**. + This backend runs int8-quantized PyTorch models on Espressif microcontrollers. It is tuned for the **ESP32-S3** and also runs on the original ESP32. It delegates quantized subgraphs to [ESP-NN](https://github.com/espressif/esp-nn),