diff --git a/.ci/scripts/test_espressif.sh b/.ci/scripts/test_espressif.sh index ca59bba695d..50a65503757 100644 --- a/.ci/scripts/test_espressif.sh +++ b/.ci/scripts/test_espressif.sh @@ -34,13 +34,24 @@ source "${ESPRESSIF_TOOLS_DIR}/setup_path.sh" python examples/espressif/export_smoke_model.py \ --output "${build_dir}/smoke.bpte" 2>&1 | tee "${build_dir}/export.log" +python examples/espressif/export_espnn_test_programs.py \ + --output-dir "${build_dir}/espnn-tests" 2>&1 | tee "${build_dir}/espnn-export.log" + +# One runtime serves the smoke test and the ESP-NN programs. quantized_ops_lib +# registers kernels outside the selected list, so the registry is sized +# explicitly instead of from the list. The quantized kernels' parallel_for +# runs inline without pthreadpool, which needs POSIX calls ESP-IDF lacks. +espnn_ops=$(tr -d '\n' < "${build_dir}/espnn-tests/operators.txt" \ + | tr ',' '\n' | grep -v '^quantized_decomposed::' | paste -sd, -) cmake --preset esp-baremetal -G Ninja -B "${build_dir}/runtime" \ -DCMAKE_TOOLCHAIN_FILE="${IDF_PATH}/tools/cmake/toolchain-esp32s3.cmake" \ -DCMAKE_BUILD_TYPE=Release \ -DEXECUTORCH_BUILD_DEVTOOLS=ON \ - -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=OFF \ - -DEXECUTORCH_SELECT_OPS_LIST=aten::add.out,aten::mul.out \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_SELECT_OPS_LIST="aten::add.out,aten::mul.out,${espnn_ops}" \ + -DMAX_KERNEL_NUM=256 \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF -DEXECUTORCH_BUILD_CPUINFO=OFF \ 2>&1 | tee "${build_dir}/runtime-configure.log" cmake --build "${build_dir}/runtime" \ 2>&1 | tee "${build_dir}/runtime-build.log" @@ -66,3 +77,45 @@ timeout 130s esp-emu --chip esp32s3 \ --net user --psram-size 8M --timeout 120s --exit-on 'Program complete.' \ 2>&1 | tee "${build_dir}/emulator.log" python examples/espressif/test_emulator.py "${build_dir}/emulator.log" + +# ESP-NN backend: run every exported program through the ExecuTorch runtime +# and the ESP-NN delegate, and check it against the bundled host reference. +# The emulator does not implement the ESP32-S3 SIMD instructions, so ESP-NN is +# built with its ANSI C kernels. +printf 'CONFIG_NN_ANSI_C=y\n' > "${build_dir}/espnn-emulator.defaults" +# Runs a command with its output in a log, and shows the log's tail on failure. +logged() { + local log="${build_dir}/$1" + shift + if ! "$@" > "${log}" 2>&1; then + tail -n 60 "${log}" + echo "FAIL: see ${log}" >&2 + return 1 + fi +} +while read -r -u 3 name atol rtol; do + logged "espnn-${name}-configure.log" \ + idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" \ + -DIDF_TARGET=esp32s3 \ + -DSDKCONFIG="${build_dir}/espnn-sdkconfig" \ + "-DSDKCONFIG_DEFAULTS=sdkconfig.defaults;${build_dir}/espnn-emulator.defaults" \ + -DET_ESPNN_BACKEND=ON \ + -DET_BUILD_DIR_PATH="${build_dir}/runtime" \ + -DET_PTE_FILE_PATH="${build_dir}/espnn-tests/${name}.bpte" \ + -DET_BUNDLE_IO=ON -DET_NUM_INFERENCES=1 \ + -DET_ATOL="${atol}" -DET_RTOL="${rtol}" reconfigure + logged "espnn-${name}-build.log" cmake --build "${build_dir}/espnn-firmware" + logged "espnn-${name}-merge.log" \ + idf.py -C examples/espressif/project -B "${build_dir}/espnn-firmware" merge-bin + logged "espnn-${name}-emulator.log" \ + timeout 90s esp-emu --chip esp32s3 \ + --firmware "${build_dir}/espnn-firmware/merged-binary.bin" \ + --elf "${build_dir}/espnn-firmware/executorch_esp_runner.elf" \ + --net user --psram-size 8M --timeout 60s --exit-on 'Program complete.' + echo -n "${name}: " + python examples/espressif/test_emulator.py "${build_dir}/espnn-${name}-emulator.log" +done 3< <(python -c ' +import json, sys +for entry in json.load(open(sys.argv[1])): + print(entry["name"], entry["atol"], entry["rtol"]) +' "${build_dir}/espnn-tests/manifest.json") diff --git a/.github/workflows/test-espressif.yml b/.github/workflows/test-espressif.yml index ddba1f2960e..5503b3845fc 100644 --- a/.github/workflows/test-espressif.yml +++ b/.github/workflows/test-espressif.yml @@ -7,6 +7,7 @@ on: - release/* pull_request: paths: + - backends/espressif/** - examples/espressif/** - tools/cmake/preset/esp_baremetal.cmake - CMakePresets.json diff --git a/backends/espressif/README.md b/backends/espressif/README.md new file mode 100644 index 00000000000..89188618fc6 --- /dev/null +++ b/backends/espressif/README.md @@ -0,0 +1,231 @@ +# ESP-NN backend for Espressif chips + +This backend is a **prototype under active development**. + +This backend runs int8-quantized PyTorch models on Espressif microcontrollers. +It is tuned for the **ESP32-S3** and also runs on the original ESP32. It +delegates quantized subgraphs to [ESP-NN](https://github.com/espressif/esp-nn), +Espressif's kernel library. ESP-NN has SIMD kernels for the ESP32-S3 and +portable C kernels for the other chips. + +A lowered model is a normal `.pte` program. `EspNNBackend` runs the int8 +partitions with ESP-NN, and ExecuTorch's portable and quantized kernels run +everything else (float ops, unsupported layers, pre- and post-processing). +Each delegate payload is a compact int8 graph (XTS3) that the delegate's +interpreter executes. A host reference interpreter reproduces the device +arithmetic, so fully delegated programs are checked bit-exactly. + +## Contents + +``` +backends/espressif/ + espnn_preprocess.py EspNNBackend: BackendDetails.preprocess -> XTS3 graph + partition/ EspNNPartitioner: int8 regions -> delegate, rest -> CPU + quantizer/ EspNNQuantizer: PT2E int8 annotations ESP-NN can execute + lowering.py to_espnn_edge(): export -> edge -> lowered program + report + serialization/ XTS3 format writer and parser, op lowering rules + reference/ bit-exact host model of the ESP-NN kernels and the CPU fallback + espnn_runtime/ ESP-IDF component: XTS3 interpreter + ExecuTorch BackendInterface + test/ pytest suite (lowering, operators, CPU fallback, parser) +examples/espressif/ + quantize_and_lower.py minimal quantize -> lower -> .pte example + export_espnn_test_programs.py the test models as BundleIO programs for the runner + camera_stream/ live camera classification on the ExecuTorch runtime +``` + +Programs run on the board with the existing +[`examples/espressif/project`](../../examples/espressif/project) runner, built +with `-DET_ESPNN_BACKEND=ON`. + +## Supported operators + +All delegated operators are int8 with per-tensor activations and per-channel +symmetric weights, as annotated by `EspNNQuantizer`. + +| Group | Operators | +|---|---| +| Convolution | `conv2d` (any stride and padding, groups 1 or depthwise, dilation through phase decomposition), `conv_transpose2d`, `linear` | +| Activations | `relu`, `relu6`, `hardtanh`, plus table lookups for `hardswish`, `hardsigmoid`, `sigmoid`, `silu`, `tanh`, `gelu`, `elu`, `leaky_relu` | +| Elementwise | `add`, `mul` (with broadcasting), `softmax` (last dim) | +| Pooling | `max_pool2d`, `avg_pool2d`, `adaptive_avg_pool2d`, `mean` over H and W | +| Resize | `upsample_nearest2d`, `upsample_bilinear2d` (`align_corners` either way) | +| Data movement | `reshape`, `view`, `flatten`, `unflatten`, `permute`, `transpose`, `squeeze`, `unsqueeze`, `slice`, `select`, `expand`, `pad`, `cat`, `stack`, `split`, `chunk`, `unbind` | + +Batch norm that follows a convolution is folded into it by PT2E +(`prepare_pt2e` / `prepare_qat_pt2e`); dropout in an exported eval-mode +model is a no-op. +Partitions can have several inputs and outputs. Anything else stays in the +program and runs on the CPU, including float ops, LSTM, layer norm and +attention matmuls. The partitioner reports what it delegated and what it left +behind. Pass `allow_cpu_fallback=False` to make leftover operators an error. + +## Quantize and lower a model + +[`examples/espressif/quantize_and_lower.py`](../../examples/espressif/quantize_and_lower.py) +contains the whole flow: + +```python +import torch +import torchvision +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + +model = torchvision.models.mobilenet_v2(weights="DEFAULT").eval() +example_inputs = (torch.randn(1, 3, 224, 224),) + +# 1. Capture the model and insert observers for the ESP-NN int8 scheme +exported = torch.export.export(model, example_inputs).module() +prepared = prepare_pt2e(exported, EspNNQuantizer()) + +# 2. Calibrate (use real samples from your dataset here) +with torch.no_grad(): + for images in calibration_batches: + prepared(images) +quantized = convert_pt2e(prepared) + +# 3. Lower: int8 regions -> EspNNBackend delegates, the rest -> CPU kernels +edge, report = to_espnn_edge(torch.export.export(quantized, example_inputs)) +print(report["partitions"], report["delegated_ops"], report["cpu_ops"]) + +# 4. Serialize the ExecuTorch program +with open("mobilenet_v2_espnn.pte", "wb") as f: + f.write(edge.to_executorch().buffer) +``` + +``` +$ python examples/espressif/quantize_and_lower.py +1 EspNN partition(s), 101 delegated operators, 0 operators left for the CPU +wrote mobilenet_v2_espnn.pte +``` + +Calibrate on a few hundred real samples; random data gives useless scales. + +## Running on the ESP32-S3 + +You need ESP-IDF (CI uses v6.1) and an activated ESP-IDF shell. +[`examples/espressif/setup.sh`](../../examples/espressif/setup.sh) installs a +matching ESP-IDF and the emulator. + +### 1. Cross-compile ExecuTorch + +Build the runtime with the `esp-baremetal` preset. Select the portable CPU +operators your program needs. The quantized kernels provide the +quantize/dequantize operators at the delegate boundaries. The program lists +its operators: + +```python +plan = edge.to_executorch().executorch_program.execution_plan[0] +print(",".join(sorted( + f"{op.name}.{op.overload}" for op in plan.operators + if not op.name.startswith("quantized_decomposed::") +))) +``` + +```bash +cmake --preset esp-baremetal -G Ninja -B cmake-out-esp \ + -DCMAKE_TOOLCHAIN_FILE="${IDF_PATH}/tools/cmake/toolchain-esp32s3.cmake" \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_DEVTOOLS=ON \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_SELECT_OPS_LIST= \ + -DMAX_KERNEL_NUM=256 \ + -DEXECUTORCH_BUILD_PTHREADPOOL=OFF -DEXECUTORCH_BUILD_CPUINFO=OFF +cmake --build cmake-out-esp --target install +``` + +- `quantized_ops_lib` registers kernels beyond the selected list, so + `MAX_KERNEL_NUM` is set explicitly instead of being sized from the + selection. +- pthreadpool needs POSIX calls that ESP-IDF doesn't provide. With it off, + the quantized kernels run single-threaded. +- The runner links `executorch_selected_kernels`, which only exists when a + selection is given. A fully delegated program with no CPU operators + still needs at least one selected operator, such as `aten::add.out`. + +### 2. Build the runner with the ESP-NN backend + +```bash +idf.py -C examples/espressif/project -B build-espnn \ + -DIDF_TARGET=esp32s3 \ + -DET_ESPNN_BACKEND=ON \ + -DET_BUILD_DIR_PATH=$PWD/cmake-out-esp \ + -DET_PTE_FILE_PATH=$PWD/mobilenet_v2_espnn.pte \ + build flash monitor +``` + +`ET_ESPNN_BACKEND` adds +[`backends/espressif/espnn_runtime`](espnn_runtime) to the project. The +component pulls in `espressif/esp-nn` from the component registry, links +against the ExecuTorch targets that `executor_runner` imports, and registers +`EspNNBackend` at startup. It defines no platform layer; the runner's +`esp_pal.cpp` provides it. + +For a BundleIO program (`.bpte`), add `-DET_BUNDLE_IO=ON` and the tolerances +`-DET_ATOL=... -DET_RTOL=...`. The runner then checks the outputs against the +bundled reference and logs `Test_result: PASS` or `FAIL`. + +Other options, in menuconfig under *ExecuTorch ESP-NN backend*: + +| Kconfig | Default | Effect | +|---|---|---| +| `ESPNN_DELEGATE_CONSTANTS_TO_PSRAM` | y | Copies weights from memory-mapped flash into PSRAM when they fit (faster than flash cache misses on large models) | +| `ESPNN_DELEGATE_DUAL_CORE` | n | Splits large convolutions across both cores | +| `ESPNN_DELEGATE_PROFILE` | n | Prints a per-operator time table after a delegate's first run | + +### The original ESP32 + +The delegate also runs on the ESP32. ESP-NN has no SIMD kernels for it and +uses portable C, about 13 million int8 multiply-accumulates per second at +240 MHz, roughly an order of magnitude slower than the S3. Most ESP32 modules +have no PSRAM, so activations must fit in about 280 KB of internal RAM. + +## Testing + +```bash +pytest backends/espressif/test +``` + +The tests export small models for every operator group, lower them, run the +XTS3 graphs through the reference interpreter and compare the result against +the quantized PyTorch model. `test_xts3_parser_host.py` builds the device +parser for the host and feeds it corrupted graphs. + +End to end, `.ci/scripts/test_espressif.sh` runs the same models through the +ExecuTorch runtime on an emulated ESP32-S3. +`examples/espressif/export_espnn_test_programs.py` exports each one as a +BundleIO program with the host reference's outputs. CI builds the runner with +`-DET_ESPNN_BACKEND=ON` for each program and runs it in `esp-emu`. Fully +delegated programs must match to float precision, which means bit-exact int8 +results. Programs with CPU regions get a tolerance for the portable kernels' +float rounding. The emulator does not implement the S3 SIMD instructions, so +this build uses ESP-NN's ANSI C kernels (`CONFIG_NN_ANSI_C=y`). + +The code follows the ExecuTorch coding style and passes the repository's +`lintrunner` checks: + +```bash +lintrunner --paths-cmd "git ls-files backends/espressif examples/espressif" +``` + +## Limitations + +- Only int8 is delegated. Float32 operators run on ExecuTorch's portable + kernels, which are slow on these chips. +- Activations are per-tensor and weights are per-channel symmetric. Dynamic + shapes are not supported, so each program has a fixed input shape. +- Attention matmuls, LSTM/GRU and normalization layers other than folded batch + norm fall back to the CPU. +- Dual-core execution only helps large convolutions, and it is off by default. +- The delegate handle comes from ExecuTorch's runtime allocator, but each + delegate allocates its activation arena, scratch buffers and optional PSRAM + weight copy with ESP-IDF's `heap_caps` allocator at `init()`, so it can + choose between internal RAM and PSRAM. Nothing is allocated during + `execute()`. + +## License + +Every source file in this backend and its examples is released under the +BSD 3-Clause license (`SPDX-License-Identifier: BSD-3-Clause`), the same +license as ExecuTorch. See the `LICENSE` file at the repository root. ESP-NN +is Apache-2.0 and is fetched from the Espressif component registry, not +vendored. diff --git a/backends/espressif/__init__.py b/backends/espressif/__init__.py new file mode 100644 index 00000000000..ecddace1659 --- /dev/null +++ b/backends/espressif/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""ESP-NN backend for Espressif ESP32 chips (ESP32-S3, ESP32, ESP32-P4, ...).""" + +from .espnn_preprocess import EspNNBackend +from .lowering import to_espnn_edge +from .partition import EspNNPartitioner +from .quantizer import EspNNQuantizer + +__all__ = ["EspNNBackend", "EspNNPartitioner", "EspNNQuantizer", "to_espnn_edge"] diff --git a/backends/espressif/espnn_preprocess.py b/backends/espressif/espnn_preprocess.py new file mode 100644 index 00000000000..1b7f7f32376 --- /dev/null +++ b/backends/espressif/espnn_preprocess.py @@ -0,0 +1,61 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import logging +from typing import List + +from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult +from executorch.exir.backend.compile_spec_schema import CompileSpec + +from .serialization.xts3 import build_graph, serialize_graph + +logger = logging.getLogger(__name__) + +BACKEND_ID = "EspNNBackend" +INT8_FORMAT_SPEC = CompileSpec("espnn_format", b"int8") + + +class EspNNBackend(BackendDetails): + """ + Ahead-of-time half of the ESP-NN delegate. + + Each partition produced by EspNNPartitioner is lowered to an XTS3 int8 + graph: ESP-NN operator records, per-channel requantization parameters, + 16-byte aligned int8/int32 constants and an input/output table. The + on-device EspNNBackend (backends/espressif/espnn_runtime) parses the blob + and executes it with ESP-NN kernels. + + Compile specs: `espnn_format` selects the payload format; only `int8` is + defined, and any other value is rejected. + """ + + @staticmethod + def preprocess(edge_program, compile_specs: List[CompileSpec]) -> PreprocessResult: + for spec in compile_specs: + if ( + spec.key == INT8_FORMAT_SPEC.key + and spec.value != INT8_FORMAT_SPEC.value + ): + raise ValueError( + f"Unsupported espnn_format {spec.value!r}; only b'int8' is defined" + ) + graph = build_graph(edge_program) + blob = serialize_graph(graph) + logger.info( + "EspNN delegate: %d tensors, %d operations, %d constant bytes, %d byte blob", + len(graph.tensors), + len(graph.operations), + len(graph.constant_data), + len(blob), + ) + return PreprocessResult(processed_bytes=blob, debug_handle_map={}) + + +__all__ = ["BACKEND_ID", "EspNNBackend", "INT8_FORMAT_SPEC"] diff --git a/backends/espressif/espnn_runtime/CMakeLists.txt b/backends/espressif/espnn_runtime/CMakeLists.txt new file mode 100644 index 00000000000..d909069f942 --- /dev/null +++ b/backends/espressif/espnn_runtime/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# ESP-IDF component providing the ExecuTorch ESP-NN delegate (EspNNBackend). It +# is meant to sit next to examples/espressif/executor_runner, which imports the +# cross-compiled ExecuTorch targets this component links against: build +# examples/espressif/project with -DET_ESPNN_BACKEND=ON. + +idf_component_register( + SRCS + "src/xts3_graph.cpp" + "src/int8_interpreter.cpp" + "src/espnn_backend.cpp" + INCLUDE_DIRS + "include" + REQUIRES + esp_timer + heap + freertos + PRIV_REQUIRES + esp-nn + WHOLE_ARCHIVE +) + +get_filename_component( + espnn_executorch_root "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE +) +target_include_directories( + ${COMPONENT_LIB} + PRIVATE "${espnn_executorch_root}/.." + "${espnn_executorch_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + ${COMPONENT_LIB} PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +# Imported by executor_runner's find_package(executorch) as a global target. +target_link_libraries(${COMPONENT_LIB} PUBLIC executorch_core) diff --git a/backends/espressif/espnn_runtime/Kconfig b/backends/espressif/espnn_runtime/Kconfig new file mode 100644 index 00000000000..8fa51b176b6 --- /dev/null +++ b/backends/espressif/espnn_runtime/Kconfig @@ -0,0 +1,24 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +menu "ExecuTorch ESP-NN backend" + + config ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + bool "Copy delegate weights from flash into PSRAM when they fit" + default y + depends on SPIRAM + + config ESPNN_DELEGATE_DUAL_CORE + bool "Split ESP-NN kernels across both cores inside delegates" + default n + + config ESPNN_DELEGATE_PROFILE + bool "Print a per-operator profile after each delegate's first run" + default n + +endmenu diff --git a/backends/espressif/espnn_runtime/idf_component.yml b/backends/espressif/espnn_runtime/idf_component.yml new file mode 100644 index 00000000000..2891ae084a6 --- /dev/null +++ b/backends/espressif/espnn_runtime/idf_component.yml @@ -0,0 +1,13 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +description: ExecuTorch ESP-NN backend runtime (XTS3 int8 interpreter and EspNNBackend delegate) +dependencies: + idf: + version: ">=5.1" + espressif/esp-nn: "^1.4.0" diff --git a/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h new file mode 100644 index 00000000000..f6047d07602 --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/espnn_backend.h @@ -0,0 +1,57 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { + +/// Backend ID that `EspNNBackend.preprocess()` writes into lowered programs. +constexpr const char* kEspNNBackendId = "EspNNBackend"; + +/** + * Counters accumulated by every EspNNBackend delegate since startup or the + * last reset_delegate_stats() call. + */ +struct DelegateStats { + /// Number of delegate instances initialized. + uint32_t delegates; + /// Number of delegate execute() calls. + uint32_t calls; + /// Total time spent in execute(), including input and output conversion. + int64_t total_us; + /// Sum of the activation arenas held by the initialized delegates. + size_t arena_bytes; +}; + +/** + * Registers EspNNBackend with the ExecuTorch runtime. + * + * Registration also happens from a static initializer when the component is + * linked whole-archive; calling this is only needed when the linker could + * drop that initializer. Safe to call more than once. + * + * @retval true The backend is registered. + */ +bool register_espnn_backend(); + +/// Returns a snapshot of the delegate counters. +DelegateStats delegate_stats(); + +/// Zeroes all delegate counters. +void reset_delegate_stats(); + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h new file mode 100644 index 00000000000..9c18ac8e63b --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/int8_interpreter.h @@ -0,0 +1,241 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +#include + +namespace executorch { +namespace backends { +namespace espressif { + +/// Which ESP-NN kernel family runs the graph. +enum class KernelMode : uint8_t { + /// Chip-optimized ESP-NN kernels (SIMD on the ESP32-S3). + OPTIMIZED = 0, + /// ESP-NN's portable ANSI C kernels, used to cross-check OPTIMIZED. + REFERENCE = 1, +}; + +/// Options for Int8Interpreter::load(). +struct Int8RuntimeOptions { + /// Kernel family used by invoke(). + KernelMode kernel_mode = KernelMode::OPTIMIZED; + /// Copy weights from memory-mapped flash into PSRAM when they fit. + bool copy_constants_to_psram = false; + /// Place the activation arena in internal RAM when it fits, else PSRAM. + bool prefer_internal_arena = true; + /// Split large convolutions across both cores. ESP-NN's worker task is + /// process-wide, so once any interpreter enables it every interpreter's + /// kernels may use it; results are bit-identical either way. + bool dual_core = false; + /// Internal RAM left free when deciding whether the arena fits there. + size_t internal_reserve_bytes = 48 * 1024; +}; + +/** + * Shape and quantization of a graph input or output. Rank-4 tensors are + * described in NCHW terms but stored on the device as NHWC. + */ +struct Int8TensorInfo { + uint32_t tensor_id; + uint8_t rank; + int32_t shape[4]; + int32_t height; + int32_t width; + int32_t channels; + size_t bytes; + float scale; + int32_t zero_point; +}; + +/** + * Runs an XTS3 int8 graph (the EspNNBackend delegate payload) with ESP-NN. + * + * The interpreter does not copy the graph: the blob passed to load() must + * stay valid until reset() or destruction. Activations live in one arena + * planned at load time, so invoke() does not allocate. + */ +class Int8Interpreter { + public: + Int8Interpreter() = default; + ~Int8Interpreter(); + Int8Interpreter(const Int8Interpreter&) = delete; + Int8Interpreter& operator=(const Int8Interpreter&) = delete; + + /** + * Validates the graph, plans activation memory and allocates the arena, + * scratch buffers and requantization tables. + * + * @param[in] blob XTS3 graph, kept by reference. + * @param[in] size Size of @p blob in bytes. + * @param[in] options Placement and kernel options. + * @retval true The graph is ready for invoke(). + * @retval false The graph is malformed or memory could not be allocated; + * the reason is printed. + */ + bool load( + const uint8_t* blob, + size_t size, + const Int8RuntimeOptions& options = {}); + + /// Frees everything allocated by load(). The interpreter can be reused. + void reset(); + + /** + * Runs every operation once on the current input data. + * + * ESP-NN keeps its scratch buffers in global state, so invoke() calls from + * all interpreters are serialized by one process-wide mutex. Concurrent + * callers on other tasks or cores wait rather than share buffers. + * + * @retval false The graph is not loaded or an operation failed. + */ + bool invoke(); + + /// Number of graph inputs. + size_t input_count() const { + return input_count_; + } + /// Number of graph outputs. + size_t output_count() const { + return output_count_; + } + /// Describes input @p index. @retval false @p index is out of range. + bool input_info(size_t index, Int8TensorInfo& info) const; + /// Describes output @p index. @retval false @p index is out of range. + bool output_info(size_t index, Int8TensorInfo& info) const; + /// Device-layout (NHWC) int8 buffer of input @p index, or nullptr. + int8_t* input_data(size_t index); + /// Device-layout (NHWC) int8 buffer of output @p index, or nullptr. + const int8_t* output_data(size_t index) const; + /// Raw data of any graph tensor, constant or activation, or nullptr. + const int8_t* tensor_data(uint32_t tensor_id) const; + + /** + * Quantizes float NCHW values into input @p index. + * + * @param[in] count Number of values; must equal the input's size. + */ + bool quantize_input_nchw(size_t index, const float* values, size_t count); + + /** + * Normalizes interleaved 8-bit RGB pixels with (x / 255 - mean) / stddev + * and quantizes them into a 3-channel input of the same width and height. + */ + bool set_input_rgb888( + size_t index, + const uint8_t* rgb, + int32_t width, + int32_t height, + const float mean[3], + const float stddev[3]); + + /// Dequantizes output @p index in device layout into @p values. + bool dequantize_output(size_t index, float* values, size_t count) const; + + /// Index of the largest int8 value of output @p index, or -1. + int32_t output_argmax(size_t index) const; + + /// Switches kernels for later invoke() calls without reloading. + void set_kernel_mode(KernelMode mode) { + options_.kernel_mode = mode; + } + KernelMode kernel_mode() const { + return options_.kernel_mode; + } + const GraphView& graph() const { + return graph_; + } + /// Duration of the last invoke() in microseconds. + int64_t last_invoke_us() const { + return last_invoke_us_; + } + /// Duration of operation @p op during the last invoke(), or 0. + int64_t op_time_us(uint32_t op) const; + /// Size of the activation arena. + size_t arena_bytes() const { + return arena_size_; + } + /// Whether the arena was placed in internal RAM rather than PSRAM. + bool arena_in_internal_ram() const { + return arena_internal_; + } + /// Size of the shared ESP-NN scratch buffer. + size_t scratch_bytes() const { + return scratch_size_; + } + /// Size of the precomputed requantization tables. + size_t parameter_bytes() const { + return params_size_; + } + /// Whether weights were copied into PSRAM by load(). + bool constants_copied() const { + return blob_copy_ != nullptr; + } + /// Prints tensor, operation and memory statistics. + void print_summary() const; + /// Prints per-operation times of the last invoke(). + void print_profile() const; + + private: + struct OpState; + + bool plan_memory(); + bool prepare_operations(); + void rebase_constants(const uint8_t* old_base, size_t size); + bool tensor_info(uint32_t tensor_id, Int8TensorInfo& info) const; + int8_t* activation(uint32_t tensor_id) const; + const int8_t* operand(uint32_t tensor_id) const; + void run_generic_conv(const OpState& s, const int8_t* input, int8_t* output) + const; + void run_dilated_by_phases( + const OpState& s, + const int8_t* input, + int8_t* output) const; + void run_copy(const OpState& s, int8_t* output) const; + void run_upsample(const OpState& s, const int8_t* input, int8_t* output) + const; + void run_transposed_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const; + bool run_operation(uint32_t index); + + Int8RuntimeOptions options_{}; + GraphView graph_{}; + bool loaded_ = false; + uint8_t* blob_copy_ = nullptr; + TensorRecord* tensors_ = nullptr; + OperationRecord* operations_ = nullptr; + OpState* op_states_ = nullptr; + size_t* tensor_offsets_ = nullptr; + uint8_t* arena_ = nullptr; + size_t arena_size_ = 0; + bool arena_internal_ = false; + void* scratch_ = nullptr; + size_t scratch_size_ = 0; + int8_t* phase_buffer_ = nullptr; + size_t phase_bytes_ = 0; + int32_t* params_ = nullptr; + size_t params_size_ = 0; + uint32_t inputs_[MAX_GRAPH_IO]{}; + uint32_t outputs_[MAX_GRAPH_IO]{}; + size_t input_count_ = 0; + size_t output_count_ = 0; + int64_t last_invoke_us_ = 0; +}; + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h new file mode 100644 index 00000000000..4fac0173531 --- /dev/null +++ b/backends/espressif/espnn_runtime/include/espnn/xts3_graph.h @@ -0,0 +1,242 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +/** + * @file + * Parser for XTS3, the int8 graph format produced by EspNNBackend.preprocess() + * (backends/espressif/serialization/xts3.py). All multi-byte fields are + * little-endian. A graph is a header, an optional input/output table + * (version 6), fixed-size tensor and operation records, a parameter region + * and a constant region. parse_graph() bounds-checks every region and record + * before any accessor reads it. + */ + +namespace executorch { +namespace backends { +namespace espressif { + +/// Graph versions accepted by this runtime; 6 adds the input/output table. +static constexpr uint32_t GRAPH_ESP_NN_INT8_VERSION = 5; +static constexpr uint32_t GRAPH_ESP_NN_INT8_IO_VERSION = 6; +static constexpr size_t GRAPH_HEADER_SIZE = 22; +static constexpr size_t GRAPH_IO_TABLE_HEADER_SIZE = 4; +static constexpr size_t MAX_GRAPH_IO = 16; +static constexpr size_t TENSOR_RECORD_SIZE = 36; +static constexpr size_t OPERATION_RECORD_SIZE = 44; +static constexpr size_t MAX_GRAPH_TENSORS = 1024; +static constexpr size_t MAX_GRAPH_OPERATIONS = 1024; + +/// TensorRecord::flags bits. +static constexpr uint8_t TENSOR_FLAG_CONSTANT = 1U << 0; +static constexpr uint8_t TENSOR_FLAG_QUANTIZED = 1U << 1; +static constexpr uint8_t TENSOR_FLAG_GRAPH_INPUT = 1U << 2; +static constexpr uint8_t TENSOR_FLAG_GRAPH_OUTPUT = 1U << 3; + +/// Serialized parameter block sizes and modes, mirrored in xts3.py. +static constexpr size_t INT8_CONV_BASE_PARAMS_SIZE = 20; +static constexpr size_t INT8_ADD_PARAMS_SIZE = 36; +static constexpr size_t INT8_PAIR_PARAMS_SIZE = 8; +static constexpr size_t INT8_POOL_PARAMS_SIZE = 16; +static constexpr size_t INT8_LUT_PARAMS_SIZE = 256; +static constexpr size_t INT8_MUL_PARAMS_SIZE = 20; +static constexpr size_t INT8_SOFTMAX_PARAMS_SIZE = 1024; +static constexpr int32_t MUL_MODE_ELEMENTWISE = 0; +static constexpr int32_t MUL_MODE_CHANNEL_BROADCAST = 1; +static constexpr int32_t SOFTMAX_OUTPUT_ZERO_POINT = -128; +static constexpr size_t INT8_PARAMETER_ALIGNMENT = 16; +static constexpr size_t COPY_HEADER_SIZE = 12; +static constexpr size_t COPY_MAX_RANK = 6; +static constexpr size_t COPY_SEGMENT_SIZE = 108; +static constexpr int32_t COPY_FLAG_FILL = 1; +static constexpr size_t UPSAMPLE_HEADER_SIZE = 20; +static constexpr int32_t UPSAMPLE_MODE_NEAREST = 0; +static constexpr int32_t UPSAMPLE_MODE_BILINEAR = 1; +static constexpr int32_t UPSAMPLE_WEIGHT_BITS = 10; +/// Largest output height or width an UPSAMPLE operation may declare. +static constexpr int32_t UPSAMPLE_MAX_EXTENT = 65535; +static constexpr size_t TRANSPOSE_CONV_PARAMS_SIZE = 52; + +/// Operation types; values are part of the file format. +enum class OpCode : uint8_t { + CONV2D = 0, + DEPTHWISE_CONV2D = 1, + RELU6 = 2, + ADD = 3, + AVGPOOL2D = 4, + LINEAR = 5, + FLATTEN = 6, + MAX_POOL2D = 7, + AVG_POOL2D = 8, + CONCAT = 9, + LUT = 10, + MUL = 11, + SOFTMAX = 12, + COPY = 13, + UPSAMPLE = 14, + TRANSPOSE_CONV2D = 15, +}; + +static constexpr uint8_t OPCODE_COUNT = 16; + +/// Tensor element types; delegated graphs only use INT8 and INT32. +enum class DataType : uint8_t { + UNKNOWN = 0, + FLOAT32 = 1, + INT8 = 2, + UINT8 = 3, + INT32 = 4, +}; + +/// Fixed graph header. +struct GraphHeader { + char magic[4]; + uint16_t version; + uint32_t num_tensors; + uint32_t num_operations; + uint32_t parameter_size; + uint32_t constant_size; +}; + +/// One tensor; shape is NCHW for rank 4, stored on the device as NHWC. +struct TensorRecord { + uint32_t tensor_id; + uint32_t constant_offset; + uint8_t dtype; + uint8_t flags; + uint8_t rank; + int32_t shape[4]; + float quant_scale; + int32_t quant_zero_point; +}; + +/// One operation with its tensor IDs and parameter block. +struct OperationRecord { + uint8_t op_id; + uint8_t opcode; + uint8_t num_inputs; + uint8_t num_outputs; + uint32_t inputs[6]; + uint32_t outputs[2]; + uint32_t parameter_offset; + uint32_t parameter_size; +}; + +/// Validated view of a graph; points into the caller's buffer. +struct GraphView { + const uint8_t* data; + size_t size; + size_t tensor_record_size; + GraphHeader header; + size_t tensor_table_offset; + size_t operation_table_offset; + size_t parameter_offset; + size_t constant_offset; + const uint8_t* parameter_data; + const uint8_t* constant_data; + uint16_t num_inputs; + uint16_t num_outputs; + const uint8_t* io_ids; +}; + +/// Decoded convolution parameters; multipliers and shifts are per channel. +struct Int8ConvParams { + int32_t kernel_h; + int32_t kernel_w; + int32_t stride_h; + int32_t stride_w; + int32_t pad_h; + int32_t pad_w; + int32_t groups; + int32_t activation_min; + int32_t activation_max; + const uint8_t* multipliers; + const uint8_t* shifts; + int32_t dilation_h; + int32_t dilation_w; +}; + +/// Decoded requantization parameters of an elementwise add. +struct Int8AddParams { + int32_t input1_mult; + int32_t input1_shift; + int32_t input2_mult; + int32_t input2_shift; + int32_t left_shift; + int32_t output_mult; + int32_t output_shift; + int32_t activation_min; + int32_t activation_max; +}; + +/// Reads a little-endian 32-bit value from unaligned memory. +uint32_t read_le_u32(const uint8_t* p); +/// Reads a little-endian 32-bit signed value from unaligned memory. +int32_t read_le_i32(const uint8_t* p); + +/** + * Validates a serialized graph and fills @p graph. + * + * Checks the magic, version, region bounds, every tensor and operation + * record, and each operation's parameter block against its tensor shapes. + * + * @param[in] data Serialized graph; must outlive @p graph. + * @param[in] size Size of @p data in bytes. + * @param[out] graph View used by the other functions. + * @retval false The graph is malformed; the reason is printed. + */ +bool parse_graph(const uint8_t* data, size_t size, GraphView& graph); +/// Whether @p version is an int8 graph version this runtime can execute. +bool is_int8_graph_version(uint32_t version); +/// Number of graph inputs. +uint32_t graph_input_count(const GraphView& graph); +/// Number of graph outputs. +uint32_t graph_output_count(const GraphView& graph); +/// Tensor ID of input @p index, or UINT32_MAX if out of range. +uint32_t graph_input_id(const GraphView& graph, uint32_t index); +/// Tensor ID of output @p index, or UINT32_MAX if out of range. +uint32_t graph_output_id(const GraphView& graph, uint32_t index); +/// Decodes tensor record @p index. @retval false Out of range. +bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor); +/// Decodes operation record @p index. @retval false Out of range. +bool read_operation( + const GraphView& graph, + uint32_t index, + OperationRecord& operation); +/// Decodes a convolution's parameter block for @p channels output channels. +bool read_int8_conv_params( + const GraphView& graph, + const OperationRecord& op, + uint32_t channels, + Int8ConvParams& params); +/// Decodes an elementwise add's parameter block. +bool read_int8_add_params( + const GraphView& graph, + const OperationRecord& op, + Int8AddParams& params); +/// Number of elements described by the tensor's shape. +size_t tensor_element_count(const TensorRecord& tensor); +/// Size of the tensor's data in bytes. +size_t tensor_data_size(const TensorRecord& tensor); +/// Constant data of @p tensor; only meaningful for TENSOR_FLAG_CONSTANT +/// tensors, whose offsets parse_graph() has checked. +const uint8_t* tensor_constant_data( + const GraphView& graph, + const TensorRecord& tensor); +/// Printable operation name. +const char* opcode_name(OpCode opcode); + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/espnn_backend.cpp b/backends/espressif/espnn_runtime/src/espnn_backend.cpp new file mode 100644 index 00000000000..06b21783611 --- /dev/null +++ b/backends/espressif/espnn_runtime/src/espnn_backend.cpp @@ -0,0 +1,299 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +using executorch::aten::ScalarType; +using executorch::aten::Tensor; +using executorch::runtime::ArrayRef; +using executorch::runtime::Backend; +using executorch::runtime::BackendExecutionContext; +using executorch::runtime::BackendInitContext; +using executorch::runtime::BackendInterface; +using executorch::runtime::CompileSpec; +using executorch::runtime::DelegateHandle; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::Result; +using executorch::runtime::Span; + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +DelegateStats g_stats{}; + +constexpr float kHalfEvenRoundingBias = 12582912.0f; + +float round_half_even_below_2p22(float value) { + return (value + kHalfEvenRoundingBias) - kHalfEvenRoundingBias; +} + +int8_t quantize(float value, float inverse_scale, int32_t zero_point) { + const float scaled = value * inverse_scale; + if (!std::isfinite(scaled)) + return scaled > 0.0f ? 127 : -128; + if (scaled > 1.0e6f) + return 127; + if (scaled < -1.0e6f) + return -128; + int32_t q = + static_cast(round_half_even_below_2p22(scaled)) + zero_point; + if (q < -128) + q = -128; + if (q > 127) + q = 127; + return static_cast(q); +} + +bool same_layout(const Tensor& tensor, const Int8TensorInfo& info) { + if (tensor.dim() != info.rank || + static_cast(tensor.numel()) != info.bytes) + return false; + for (uint8_t d = 0; d < info.rank; ++d) { + if (tensor.size(d) != info.shape[d]) + return false; + } + return executorch::runtime::is_contiguous_dim_order( + tensor.dim_order().data(), tensor.dim_order().size()); +} + +bool load_input( + Int8Interpreter& interpreter, + size_t index, + const Tensor& tensor) { + Int8TensorInfo info{}; + if (!interpreter.input_info(index, info) || !same_layout(tensor, info)) + return false; + int8_t* destination = interpreter.input_data(index); + const size_t plane = static_cast(info.height) * info.width; + const size_t channels = static_cast(info.channels); + const bool nchw = info.rank == 4; + if (tensor.scalar_type() == ScalarType::Float) { + const float* source = tensor.const_data_ptr(); + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + if (!nchw) { + for (size_t i = 0; i < info.bytes; ++i) + destination[i] = quantize(source[i], inverse, info.zero_point); + return true; + } + for (size_t p = 0; p < plane; ++p) { + for (size_t c = 0; c < channels; ++c) { + destination[p * channels + c] = + quantize(source[c * plane + p], inverse, info.zero_point); + } + } + return true; + } + if (tensor.scalar_type() == ScalarType::Char) { + const int8_t* source = tensor.const_data_ptr(); + if (!nchw) { + std::memcpy(destination, source, info.bytes); + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) + destination[p * channels + c] = source[c * plane + p]; + } + return true; + } + return false; +} + +bool store_output( + const Int8Interpreter& interpreter, + size_t index, + Tensor& tensor) { + Int8TensorInfo info{}; + if (!interpreter.output_info(index, info) || !same_layout(tensor, info)) + return false; + const int8_t* source = interpreter.output_data(index); + const size_t plane = static_cast(info.height) * info.width; + const size_t channels = static_cast(info.channels); + const bool nchw = info.rank == 4; + if (tensor.scalar_type() == ScalarType::Float) { + float* destination = tensor.mutable_data_ptr(); + if (!nchw) { + for (size_t i = 0; i < info.bytes; ++i) { + destination[i] = + static_cast(source[i] - info.zero_point) * info.scale; + } + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) { + destination[c * plane + p] = + static_cast(source[p * channels + c] - info.zero_point) * + info.scale; + } + } + return true; + } + if (tensor.scalar_type() == ScalarType::Char) { + int8_t* destination = tensor.mutable_data_ptr(); + if (!nchw) { + std::memcpy(destination, source, info.bytes); + return true; + } + for (size_t c = 0; c < channels; ++c) { + for (size_t p = 0; p < plane; ++p) + destination[c * plane + p] = source[p * channels + c]; + } + return true; + } + return false; +} + +struct DelegateState { + Int8Interpreter interpreter; + bool profiled = false; +}; + +class EspNNBackendInterface final : public BackendInterface { + public: + bool is_available() const override { + return true; + } + + Result init( + BackendInitContext& context, + FreeableBuffer* processed, + ArrayRef) const override { + void* memory = context.get_runtime_allocator()->allocate( + sizeof(DelegateState), alignof(DelegateState)); + if (memory == nullptr) + return Error::MemoryAllocationFailed; + DelegateState* state = new (memory) DelegateState(); + Int8RuntimeOptions options{}; +#if CONFIG_ESPNN_DELEGATE_CONSTANTS_TO_PSRAM + options.copy_constants_to_psram = true; +#endif +#if CONFIG_ESPNN_DELEGATE_DUAL_CORE + options.dual_core = true; +#endif + if (!state->interpreter.load( + static_cast(processed->data()), + processed->size(), + options)) { + state->~DelegateState(); + return Error::DelegateInvalidCompatibility; + } + if (state->interpreter.constants_copied()) + processed->Free(); + ++g_stats.delegates; + g_stats.arena_bytes += state->interpreter.arena_bytes(); + return state; + } + + Error execute( + BackendExecutionContext&, + DelegateHandle* handle, + Span args) const override { + DelegateState* state = static_cast(handle); + Int8Interpreter& interpreter = state->interpreter; + const size_t inputs = interpreter.input_count(); + const size_t outputs = interpreter.output_count(); + if (args.size() != inputs + outputs) { + std::printf( + "EspNN delegate: expected %u arguments, got %u\n", + static_cast(inputs + outputs), + static_cast(args.size())); + return Error::InvalidArgument; + } + const int64_t started = esp_timer_get_time(); + for (size_t i = 0; i < inputs; ++i) { + if (!args[i]->isTensor() || + !load_input(interpreter, i, args[i]->toTensor())) { + std::printf( + "EspNN delegate: input %u has an unsupported dtype, shape or dim order\n", + static_cast(i)); + return Error::InvalidArgument; + } + } + if (!interpreter.invoke()) + return Error::Internal; + for (size_t o = 0; o < outputs; ++o) { + if (!args[inputs + o]->isTensor()) + return Error::InvalidArgument; + Tensor tensor = args[inputs + o]->toTensor(); + if (!store_output(interpreter, o, tensor)) { + std::printf( + "EspNN delegate: output %u has an unsupported dtype, shape or dim order\n", + static_cast(o)); + return Error::InvalidArgument; + } + } + ++g_stats.calls; + g_stats.total_us += esp_timer_get_time() - started; +#if CONFIG_ESPNN_DELEGATE_PROFILE + if (!state->profiled) { + state->profiled = true; + std::printf( + "EspNN delegate profile (%u ops, %lld us):\n", + static_cast(interpreter.graph().header.num_operations), + static_cast(interpreter.last_invoke_us())); + interpreter.print_profile(); + } +#endif + return Error::Ok; + } + + void destroy(DelegateHandle* handle) const override { + if (handle != nullptr) + static_cast(handle)->~DelegateState(); + } +}; + +EspNNBackendInterface g_backend; +bool g_registered = executorch::runtime::register_backend( + Backend{kEspNNBackendId, &g_backend}) == Error::Ok; + +} // namespace + +[[maybe_unused]] bool register_espnn_backend() { + if (!g_registered) { + g_registered = executorch::runtime::register_backend( + Backend{kEspNNBackendId, &g_backend}) == Error::Ok; + } + return g_registered; +} + +DelegateStats delegate_stats() { + return g_stats; +} + +[[maybe_unused]] void reset_delegate_stats() { + g_stats = {}; +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/int8_interpreter.cpp b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp new file mode 100644 index 00000000000..391574781c2 --- /dev/null +++ b/backends/espressif/espnn_runtime/src/int8_interpreter.cpp @@ -0,0 +1,1790 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +constexpr size_t ARENA_ALIGNMENT = 16; +constexpr int64_t YIELD_INTERVAL_US = 1000000; + +SemaphoreHandle_t invoke_mutex() { + static StaticSemaphore_t storage; + static SemaphoreHandle_t handle = xSemaphoreCreateMutexStatic(&storage); + return handle; +} + +class InvokeLock { + public: + InvokeLock() : handle_(invoke_mutex()) { + xSemaphoreTake(handle_, portMAX_DELAY); + } + ~InvokeLock() { + xSemaphoreGive(handle_); + } + InvokeLock(const InvokeLock&) = delete; + InvokeLock& operator=(const InvokeLock&) = delete; + + private: + SemaphoreHandle_t handle_; +}; + +size_t align_up(size_t value, size_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +void* allocate_preferring_internal( + size_t bytes, + bool prefer_internal, + size_t reserve, + bool* internal) { + if (bytes == 0) + return nullptr; + if (prefer_internal) { + const size_t largest = + heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (largest > bytes + reserve) { + void* memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (memory != nullptr) { + if (internal != nullptr) + *internal = true; + return memory; + } + } + } + void* memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (memory != nullptr) { + if (internal != nullptr) + *internal = false; + return memory; + } + memory = heap_caps_aligned_alloc( + ARENA_ALIGNMENT, bytes, MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (internal != nullptr) + *internal = memory != nullptr; + return memory; +} + +int8_t clamp_to_int8(int32_t value) { + if (value < -128) + return -128; + if (value > 127) + return 127; + return static_cast(value); +} + +int32_t rounding_high_mul(int32_t a, int32_t b) { + if (a == b && a == INT32_MIN) + return INT32_MAX; + int64_t nudge = 1 << 30; + if ((a < 0) ^ (b < 0)) + nudge = 1 - nudge; + const int64_t product = static_cast(a) * b + nudge; + const int64_t to_add = product < 0 ? (1LL << 31) - 1 : 0; + return static_cast((product + to_add) >> 31); +} + +int32_t rounding_shift_right(int32_t value, int32_t exponent) { + const int32_t mask = static_cast((1LL << exponent) - 1); + const int32_t remainder = value & mask; + int32_t result = value >> exponent; + const int32_t threshold = (mask >> 1) + (result < 0 ? 1 : 0); + if (remainder > threshold) + result += 1; + return result; +} + +int32_t requantize(int32_t value, int32_t multiplier, int32_t shift) { + const int32_t left = shift > 0 ? shift : 0; + const int32_t right = shift > 0 ? 0 : -shift; + const int32_t shifted = + static_cast(static_cast(value) << left); + return rounding_shift_right(rounding_high_mul(shifted, multiplier), right); +} + +int8_t quantize_value(float value, float inverse_scale, int32_t zero_point) { + const float scaled = value * inverse_scale; + if (!std::isfinite(scaled)) + return clamp_to_int8(scaled > 0.0f ? 127 : -128); + if (scaled > 1.0e6f) + return 127; + if (scaled < -1.0e6f) + return -128; + return clamp_to_int8( + static_cast(std::nearbyintf(scaled)) + zero_point); +} + +} // namespace + +struct Int8Interpreter::OpState { + OpCode opcode; + uint32_t input0; + uint32_t input1; + uint32_t output; + uint8_t input_count; + uint32_t inputs[6]; + int32_t pool[6]; + const int8_t* weights; + const int32_t* bias; + data_dims_t input_dims; + data_dims_t filter_dims; + data_dims_t output_dims; + conv_params_t conv; + dw_conv_params_t dw; + quant_data_t quant; + int32_t* multipliers; + int32_t* shifts; + Int8AddParams add; + int32_t value0; + int32_t value1; + int32_t input_zero_point; + int32_t output_zero_point; + uint32_t elements; + int32_t rows; + int32_t cols; + bool transpose; + bool generic; + const uint8_t* table; + int32_t dilation_h; + int32_t dilation_w; + int32_t batches; + int64_t total_us; + int64_t last_us; + uint32_t calls; +}; + +Int8Interpreter::~Int8Interpreter() { + reset(); +} + +void Int8Interpreter::reset() { + if (arena_ != nullptr) + heap_caps_free(arena_); + if (scratch_ != nullptr) + heap_caps_free(scratch_); + if (phase_buffer_ != nullptr) + heap_caps_free(phase_buffer_); + phase_buffer_ = nullptr; + phase_bytes_ = 0; + if (params_ != nullptr) + heap_caps_free(params_); + if (blob_copy_ != nullptr) + heap_caps_free(blob_copy_); + if (tensors_ != nullptr) + heap_caps_free(tensors_); + if (operations_ != nullptr) + heap_caps_free(operations_); + if (op_states_ != nullptr) + heap_caps_free(op_states_); + if (tensor_offsets_ != nullptr) + heap_caps_free(tensor_offsets_); + arena_ = nullptr; + scratch_ = nullptr; + params_ = nullptr; + blob_copy_ = nullptr; + tensors_ = nullptr; + operations_ = nullptr; + op_states_ = nullptr; + tensor_offsets_ = nullptr; + arena_size_ = 0; + scratch_size_ = 0; + params_size_ = 0; + input_count_ = 0; + output_count_ = 0; + loaded_ = false; + graph_ = {}; +} + +bool Int8Interpreter::load( + const uint8_t* blob, + size_t size, + const Int8RuntimeOptions& options) { + reset(); + options_ = options; + if (!parse_graph(blob, size, graph_)) { + std::printf("ERROR: int8 runtime rejected the model blob\n"); + return false; + } + if (!is_int8_graph_version(graph_.header.version)) { + std::printf( + "ERROR: model is XTS3 v%u; the int8 runtime requires v%u or v%u\n", + static_cast(graph_.header.version), + static_cast(GRAPH_ESP_NN_INT8_VERSION), + static_cast(GRAPH_ESP_NN_INT8_IO_VERSION)); + return false; + } + const bool misaligned = + (reinterpret_cast(graph_.constant_data) & 15U) != 0; + if (misaligned) { + blob_copy_ = static_cast(allocate_preferring_internal( + size, false, options_.internal_reserve_bytes, nullptr)); + if (blob_copy_ == nullptr) { + std::printf( + "ERROR: unable to copy %u-byte misaligned model into RAM\n", + static_cast(size)); + return false; + } + std::memcpy(blob_copy_, blob, size); + if (!parse_graph(blob_copy_, size, graph_)) + return false; + std::printf( + "WARNING: model constants were not 16-byte aligned; copied to RAM\n"); + } + + const uint32_t tensor_count = graph_.header.num_tensors; + const uint32_t op_count = graph_.header.num_operations; + tensors_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(TensorRecord), MALLOC_CAP_8BIT)); + operations_ = static_cast( + heap_caps_calloc(op_count, sizeof(OperationRecord), MALLOC_CAP_8BIT)); + op_states_ = static_cast( + heap_caps_calloc(op_count, sizeof(OpState), MALLOC_CAP_8BIT)); + tensor_offsets_ = static_cast( + heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + if (tensors_ == nullptr || operations_ == nullptr || op_states_ == nullptr || + tensor_offsets_ == nullptr) { + std::printf("ERROR: unable to allocate interpreter tables\n"); + reset(); + return false; + } + for (uint32_t i = 0; i < tensor_count; ++i) + read_tensor(graph_, i, tensors_[i]); + input_count_ = graph_input_count(graph_); + output_count_ = graph_output_count(graph_); + if (input_count_ > MAX_GRAPH_IO || output_count_ > MAX_GRAPH_IO) { + std::printf( + "ERROR: model has more than %u inputs or outputs\n", + static_cast(MAX_GRAPH_IO)); + reset(); + return false; + } + for (uint32_t i = 0; i < input_count_; ++i) + inputs_[i] = graph_input_id(graph_, i); + for (uint32_t i = 0; i < output_count_; ++i) + outputs_[i] = graph_output_id(graph_, i); + for (uint32_t i = 0; i < op_count; ++i) + read_operation(graph_, i, operations_[i]); + + if (!plan_memory() || !prepare_operations()) { + reset(); + return false; + } + if (options_.copy_constants_to_psram && blob_copy_ == nullptr && + heap_caps_get_total_size(MALLOC_CAP_SPIRAM) != 0) { + const size_t largest = + heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (largest >= size + 64 * 1024) { + blob_copy_ = static_cast(heap_caps_aligned_alloc( + ARENA_ALIGNMENT, size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + } + if (blob_copy_ == nullptr) { + std::printf( + "WARNING: not enough PSRAM to copy %u model bytes; reading weights from flash\n", + static_cast(size)); + } else { + std::memcpy(blob_copy_, graph_.data, size); + const uint8_t* old_base = graph_.data; + if (!parse_graph(blob_copy_, size, graph_)) { + reset(); + return false; + } + rebase_constants(old_base, size); + } + } + if (options_.dual_core) { +#if ESP_NN_DUAL_CORE_SUPPORTED + esp_nn_dual_core_enable(); +#else + std::printf("WARNING: dual-core ESP-NN is not supported on this target\n"); +#endif + } + loaded_ = true; + return true; +} + +bool Int8Interpreter::plan_memory() { + const uint32_t tensor_count = graph_.header.num_tensors; + const int32_t op_count = static_cast(graph_.header.num_operations); + int32_t* first = static_cast( + heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + int32_t* last = static_cast( + heap_caps_calloc(tensor_count, sizeof(int32_t), MALLOC_CAP_8BIT)); + size_t* bytes = static_cast( + heap_caps_calloc(tensor_count, sizeof(size_t), MALLOC_CAP_8BIT)); + bool* planned = static_cast( + heap_caps_calloc(tensor_count, sizeof(bool), MALLOC_CAP_8BIT)); + struct PlanBuffers { + void* buffers[4]; + ~PlanBuffers() { + for (void* buffer : buffers) { + if (buffer != nullptr) + heap_caps_free(buffer); + } + } + } plan_buffers{{first, last, bytes, planned}}; + if (first == nullptr || last == nullptr || bytes == nullptr || + planned == nullptr) { + std::printf("ERROR: unable to allocate memory-planner tables\n"); + return false; + } + for (uint32_t i = 0; i < tensor_count; ++i) { + first[i] = + (tensors_[i].flags & TENSOR_FLAG_GRAPH_INPUT) != 0 ? -1 : INT32_MAX; + last[i] = + (tensors_[i].flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 ? op_count : -1; + bytes[i] = align_up(tensor_data_size(tensors_[i]), ARENA_ALIGNMENT); + tensor_offsets_[i] = SIZE_MAX; + } + for (int32_t op = 0; op < op_count; ++op) { + const OperationRecord& record = operations_[op]; + for (uint8_t j = 0; j < record.num_inputs; ++j) { + const uint32_t id = record.inputs[j]; + if ((tensors_[id].flags & TENSOR_FLAG_CONSTANT) == 0 && last[id] < op) + last[id] = op; + } + for (uint8_t j = 0; j < record.num_outputs; ++j) { + const uint32_t id = record.outputs[j]; + if (first[id] > op) + first[id] = op; + if (last[id] < op) + last[id] = op; + } + } + + size_t arena_size = 0; + for (uint32_t step = 0; step < tensor_count; ++step) { + int32_t best = -1; + for (uint32_t i = 0; i < tensor_count; ++i) { + if (planned[i] || (tensors_[i].flags & TENSOR_FLAG_CONSTANT) != 0) + continue; + if (first[i] == INT32_MAX) + continue; + if (best < 0 || bytes[i] > bytes[best]) + best = static_cast(i); + } + if (best < 0) + break; + planned[best] = true; + size_t candidate = 0; + bool moved = true; + while (moved) { + moved = false; + for (uint32_t j = 0; j < tensor_count; ++j) { + if (j == static_cast(best) || tensor_offsets_[j] == SIZE_MAX) + continue; + const bool lifetime_overlap = + first[j] <= last[best] && first[best] <= last[j]; + if (!lifetime_overlap) + continue; + const size_t other_begin = tensor_offsets_[j]; + const size_t other_end = other_begin + bytes[j]; + if (candidate < other_end && other_begin < candidate + bytes[best]) { + candidate = other_end; + moved = true; + } + } + } + tensor_offsets_[best] = candidate; + if (candidate + bytes[best] > arena_size) + arena_size = candidate + bytes[best]; + } + for (uint32_t i = 0; i < tensor_count; ++i) { + if ((tensors_[i].flags & TENSOR_FLAG_CONSTANT) == 0 && + tensor_offsets_[i] == SIZE_MAX) { + std::printf( + "ERROR: tensor %u is never produced or consumed\n", + static_cast(i)); + return false; + } + } + + arena_size_ = arena_size; + arena_ = static_cast(allocate_preferring_internal( + arena_size_, + options_.prefer_internal_arena, + options_.internal_reserve_bytes, + &arena_internal_)); + if (arena_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte activation arena\n", + static_cast(arena_size_)); + return false; + } + std::memset(arena_, 0, arena_size_); + return true; +} + +bool Int8Interpreter::prepare_operations() { + const uint32_t op_count = graph_.header.num_operations; + size_t param_words = 0; + for (uint32_t i = 0; i < op_count; ++i) { + const OperationRecord& op = operations_[i]; + const OpCode code = static_cast(op.opcode); + if (code == OpCode::CONV2D || code == OpCode::DEPTHWISE_CONV2D || + code == OpCode::LINEAR || code == OpCode::TRANSPOSE_CONV2D) { + param_words += 2U * tensor_element_count(tensors_[op.inputs[2]]); + } else if (code == OpCode::SOFTMAX) { + param_words += 256U; + } + } + params_size_ = param_words * sizeof(int32_t); + if (params_size_ != 0) { + params_ = static_cast(allocate_preferring_internal( + params_size_, true, options_.internal_reserve_bytes, nullptr)); + if (params_ == nullptr) { + std::printf("ERROR: unable to allocate requantization tables\n"); + return false; + } + } + + int32_t* cursor = params_; + size_t scratch_size = 0; + for (uint32_t i = 0; i < op_count; ++i) { + const OperationRecord& op = operations_[i]; + OpState& state = op_states_[i]; + state = {}; + state.opcode = static_cast(op.opcode); + state.input0 = op.inputs[0]; + state.input1 = op.num_inputs > 1 ? op.inputs[1] : 0; + state.output = op.outputs[0]; + const TensorRecord& input = tensors_[op.inputs[0]]; + const TensorRecord& output = tensors_[op.outputs[0]]; + state.input_zero_point = input.quant_zero_point; + state.output_zero_point = output.quant_zero_point; + state.elements = static_cast(tensor_element_count(output)); + const uint8_t* p = graph_.parameter_data + op.parameter_offset; + + switch (state.opcode) { + case OpCode::CONV2D: + case OpCode::DEPTHWISE_CONV2D: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + const uint32_t channels = static_cast(output.shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph_, op, channels, params)) + return false; + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + state.multipliers = cursor; + state.shifts = cursor + channels; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(params.multipliers + c * 4); + state.shifts[c] = read_le_i32(params.shifts + c * 4); + } + cursor += 2U * channels; + state.quant.mult = state.multipliers; + state.quant.shift = state.shifts; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + state.dilation_h = params.dilation_h; + state.dilation_w = params.dilation_w; + state.generic = params.dilation_h != 1 || params.dilation_w != 1; + state.pool[0] = params.kernel_h; + state.pool[1] = params.kernel_w; + state.pool[2] = params.stride_h; + state.pool[3] = params.stride_w; + state.pool[4] = params.pad_h; + state.pool[5] = params.pad_w; + state.value0 = params.activation_min; + state.value1 = params.activation_max; + state.rows = params.groups; + if (state.generic) { + if (params.stride_h != 1 || params.stride_w != 1) + break; + state.transpose = true; + const int32_t phase_h = + (output.shape[2] + params.dilation_h - 1) / params.dilation_h; + const int32_t phase_w = + (output.shape[3] + params.dilation_w - 1) / params.dilation_w; + data_dims_t tile_in{ + phase_w + params.kernel_w - 1, + phase_h + params.kernel_h - 1, + input.shape[1], + 1}; + data_dims_t tile_out{phase_w, phase_h, output.shape[1], 1}; + const size_t tile_bytes = static_cast(tile_in.width) * + tile_in.height * tile_in.channels + + static_cast(phase_w) * phase_h * output.shape[1]; + if (tile_bytes > phase_bytes_) + phase_bytes_ = tile_bytes; + if (state.opcode == OpCode::CONV2D) { + state.filter_dims = { + params.kernel_w, params.kernel_h, weight.shape[1], 1}; + state.conv.in_offset = -input.quant_zero_point; + state.conv.out_offset = output.quant_zero_point; + state.conv.stride = {1, 1}; + state.conv.padding = {0, 0}; + state.conv.dilation = {1, 1}; + state.conv.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_conv_scratch_size( + &tile_in, &state.filter_dims, &tile_out, &state.conv); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } else { + state.filter_dims = { + params.kernel_w, params.kernel_h, output.shape[1], 1}; + state.dw.in_offset = -input.quant_zero_point; + state.dw.out_offset = output.quant_zero_point; + state.dw.ch_mult = output.shape[1] / input.shape[1]; + state.dw.stride = {1, 1}; + state.dw.padding = {0, 0}; + state.dw.dilation = {1, 1}; + state.dw.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_depthwise_conv_scratch_size( + &tile_in, &state.filter_dims, &tile_out, &state.dw); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } + break; + } + if (state.opcode == OpCode::CONV2D) { + state.filter_dims = { + params.kernel_w, params.kernel_h, weight.shape[1], 1}; + state.conv.in_offset = -input.quant_zero_point; + state.conv.out_offset = output.quant_zero_point; + state.conv.stride = {params.stride_w, params.stride_h}; + state.conv.padding = {params.pad_w, params.pad_h}; + state.conv.dilation = {1, 1}; + state.conv.activation = { + params.activation_min, params.activation_max}; + const int needed = esp_nn_get_conv_scratch_size( + &state.input_dims, + &state.filter_dims, + &state.output_dims, + &state.conv); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } else { + state.filter_dims = { + params.kernel_w, params.kernel_h, output.shape[1], 1}; + state.dw.in_offset = -input.quant_zero_point; + state.dw.out_offset = output.quant_zero_point; + state.dw.ch_mult = output.shape[1] / input.shape[1]; + state.dw.stride = {params.stride_w, params.stride_h}; + state.dw.padding = {params.pad_w, params.pad_h}; + state.dw.dilation = {1, 1}; + state.dw.activation = {params.activation_min, params.activation_max}; + const int needed = esp_nn_get_depthwise_conv_scratch_size( + &state.input_dims, + &state.filter_dims, + &state.output_dims, + &state.dw); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + } + break; + } + case OpCode::RELU6: + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + break; + case OpCode::ADD: + if (!read_int8_add_params(graph_, op, state.add)) + return false; + state.value0 = tensors_[op.inputs[1]].quant_zero_point; + break; + case OpCode::COPY: + state.input_count = op.num_inputs; + for (uint8_t k = 0; k < op.num_inputs; ++k) + state.inputs[k] = op.inputs[k]; + state.table = p; + break; + case OpCode::UPSAMPLE: + state.table = p; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + state.value0 = read_le_i32(p); + break; + case OpCode::TRANSPOSE_CONV2D: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + const uint32_t channels = static_cast(output.shape[1]); + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + for (int k = 0; k < 6; ++k) + state.pool[k] = read_le_i32(p + 4 * k); + state.dilation_h = read_le_i32(p + 32); + state.dilation_w = read_le_i32(p + 36); + state.value0 = read_le_i32(p + 44); + state.value1 = read_le_i32(p + 48); + state.multipliers = cursor; + state.shifts = cursor + channels; + const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + const uint8_t* shifts = multipliers + channels * 4U; + for (uint32_t c = 0; c < channels; ++c) { + state.multipliers[c] = read_le_i32(multipliers + c * 4); + state.shifts[c] = read_le_i32(shifts + c * 4); + } + cursor += 2U * channels; + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + break; + } + case OpCode::AVGPOOL2D: { + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + const int32_t needed = esp_nn_get_mean_scratch_size( + input.shape[2], input.shape[3], input.shape[1]); + if (needed > 0 && static_cast(needed) > scratch_size) + scratch_size = needed; + break; + } + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: + for (int k = 0; k < 6; ++k) + state.pool[k] = p[k]; + state.value0 = read_le_i32(p + 8); + state.value1 = read_le_i32(p + 12); + state.input_dims = {input.shape[3], input.shape[2], input.shape[1], 1}; + state.output_dims = { + output.shape[3], output.shape[2], output.shape[1], 1}; + break; + case OpCode::LUT: + state.weights = reinterpret_cast(p); + break; + case OpCode::MUL: + state.value0 = tensors_[op.inputs[1]].quant_zero_point; + state.add.output_mult = read_le_i32(p); + state.add.output_shift = read_le_i32(p + 4); + state.add.activation_min = read_le_i32(p + 8); + state.add.activation_max = read_le_i32(p + 12); + state.transpose = read_le_i32(p + 16) == MUL_MODE_CHANNEL_BROADCAST; + state.cols = input.rank == 4 ? input.shape[1] + : static_cast(state.elements); + state.rows = static_cast(state.elements) / state.cols; + break; + case OpCode::SOFTMAX: + state.multipliers = cursor; + for (int k = 0; k < 256; ++k) + state.multipliers[k] = static_cast(read_le_u32(p + 4 * k)); + cursor += 256; + state.cols = input.shape[1]; + state.rows = static_cast(state.elements) / state.cols; + break; + case OpCode::CONCAT: + state.input_count = op.num_inputs; + for (uint8_t k = 0; k < op.num_inputs; ++k) + state.inputs[k] = op.inputs[k]; + state.rows = output.rank == 4 ? output.shape[2] * output.shape[3] : 1; + state.cols = output.shape[1]; + break; + case OpCode::FLATTEN: + state.transpose = + input.rank == 4 && input.shape[2] * input.shape[3] > 1; + state.input_dims = input.rank == 4 + ? data_dims_t{input.shape[3], input.shape[2], input.shape[1], 1} + : data_dims_t{ + 1, 1, static_cast(tensor_element_count(input)), 1}; + break; + case OpCode::LINEAR: { + const TensorRecord& weight = tensors_[op.inputs[1]]; + const TensorRecord& bias = tensors_[op.inputs[2]]; + state.rows = weight.shape[0]; + state.cols = weight.shape[1]; + state.batches = static_cast( + tensor_element_count(input) / static_cast(weight.shape[1])); + state.weights = reinterpret_cast( + tensor_constant_data(graph_, weight)); + state.bias = reinterpret_cast( + tensor_constant_data(graph_, bias)); + state.value0 = read_le_i32(p); + state.value1 = read_le_i32(p + 4); + state.multipliers = cursor; + state.shifts = cursor + state.rows; + const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; + const uint8_t* shifts = + multipliers + static_cast(state.rows) * 4U; + for (int32_t r = 0; r < state.rows; ++r) { + state.multipliers[r] = read_le_i32(multipliers + r * 4); + state.shifts[r] = read_le_i32(shifts + r * 4); + } + cursor += 2 * state.rows; + break; + } + default: + return false; + } + } + + if (phase_bytes_ != 0) { + phase_buffer_ = static_cast(allocate_preferring_internal( + phase_bytes_, true, options_.internal_reserve_bytes, nullptr)); + if (phase_buffer_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte dilated-convolution tile buffer\n", + static_cast(phase_bytes_)); + return false; + } + } + scratch_size_ = scratch_size; + if (scratch_size_ != 0) { + scratch_ = allocate_preferring_internal( + scratch_size_, true, options_.internal_reserve_bytes, nullptr); + if (scratch_ == nullptr) { + std::printf( + "ERROR: unable to allocate %u-byte ESP-NN scratch buffer\n", + static_cast(scratch_size_)); + return false; + } + } + return true; +} + +void Int8Interpreter::rebase_constants(const uint8_t* old_base, size_t size) { + const uint8_t* old_end = old_base + size; + auto rebase = [&](const void* pointer) -> const uint8_t* { + const uint8_t* bytes = static_cast(pointer); + if (bytes >= old_base && bytes < old_end) + return graph_.data + (bytes - old_base); + return bytes; + }; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + OpState& state = op_states_[i]; + if (state.weights != nullptr) + state.weights = reinterpret_cast(rebase(state.weights)); + if (state.bias != nullptr) + state.bias = reinterpret_cast(rebase(state.bias)); + if (state.table != nullptr) + state.table = rebase(state.table); + } +} + +int8_t* Int8Interpreter::activation(uint32_t tensor_id) const { + return reinterpret_cast(arena_ + tensor_offsets_[tensor_id]); +} + +const int8_t* Int8Interpreter::operand(uint32_t tensor_id) const { + if ((tensors_[tensor_id].flags & TENSOR_FLAG_CONSTANT) != 0) { + return reinterpret_cast( + tensor_constant_data(graph_, tensors_[tensor_id])); + } + return activation(tensor_id); +} + +const int8_t* Int8Interpreter::tensor_data(uint32_t tensor_id) const { + if (!loaded_ || tensor_id >= graph_.header.num_tensors) + return nullptr; + if ((tensors_[tensor_id].flags & TENSOR_FLAG_CONSTANT) != 0) { + return reinterpret_cast( + tensor_constant_data(graph_, tensors_[tensor_id])); + } + return activation(tensor_id); +} + +bool Int8Interpreter::tensor_info(uint32_t tensor_id, Int8TensorInfo& info) + const { + if (!loaded_ || tensor_id >= graph_.header.num_tensors) + return false; + const TensorRecord& tensor = tensors_[tensor_id]; + info = {}; + info.tensor_id = tensor_id; + info.rank = tensor.rank; + std::memcpy(info.shape, tensor.shape, sizeof(info.shape)); + info.bytes = tensor_data_size(tensor); + info.scale = tensor.quant_scale; + info.zero_point = tensor.quant_zero_point; + if (tensor.rank == 4) { + info.height = tensor.shape[2]; + info.width = tensor.shape[3]; + info.channels = tensor.shape[1]; + } else { + info.height = 1; + info.width = 1; + info.channels = static_cast(tensor_element_count(tensor)); + } + return true; +} + +bool Int8Interpreter::input_info(size_t index, Int8TensorInfo& info) const { + return index < input_count_ && tensor_info(inputs_[index], info); +} + +bool Int8Interpreter::output_info(size_t index, Int8TensorInfo& info) const { + return index < output_count_ && tensor_info(outputs_[index], info); +} + +int8_t* Int8Interpreter::input_data(size_t index) { + return loaded_ && index < input_count_ ? activation(inputs_[index]) : nullptr; +} + +const int8_t* Int8Interpreter::output_data(size_t index) const { + return loaded_ && index < output_count_ ? activation(outputs_[index]) + : nullptr; +} + +bool Int8Interpreter::quantize_input_nchw( + size_t index, + const float* values, + size_t count) { + Int8TensorInfo info{}; + if (values == nullptr || !input_info(index, info) || count != info.bytes) + return false; + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + int8_t* destination = input_data(index); + const size_t plane = static_cast(info.height) * info.width; + for (int32_t c = 0; c < info.channels; ++c) { + for (size_t s = 0; s < plane; ++s) { + destination[s * info.channels + c] = + quantize_value(values[c * plane + s], inverse, info.zero_point); + } + } + return true; +} + +bool Int8Interpreter::set_input_rgb888( + size_t index, + const uint8_t* rgb, + int32_t width, + int32_t height, + const float mean[3], + const float stddev[3]) { + Int8TensorInfo info{}; + if (rgb == nullptr || !input_info(index, info) || info.channels != 3 || + info.width != width || info.height != height) + return false; + const float inverse = + static_cast(1.0 / static_cast(info.scale)); + int8_t lut[3][256]; + for (int c = 0; c < 3; ++c) { + for (int v = 0; v < 256; ++v) { + const float normalized = + (static_cast(v) / 255.0f - mean[c]) / stddev[c]; + lut[c][v] = quantize_value(normalized, inverse, info.zero_point); + } + } + int8_t* destination = input_data(index); + const size_t pixels = static_cast(width) * height; + for (size_t i = 0; i < pixels; ++i) { + destination[i * 3 + 0] = lut[0][rgb[i * 3 + 0]]; + destination[i * 3 + 1] = lut[1][rgb[i * 3 + 1]]; + destination[i * 3 + 2] = lut[2][rgb[i * 3 + 2]]; + } + return true; +} + +bool Int8Interpreter::dequantize_output( + size_t index, + float* values, + size_t count) const { + Int8TensorInfo info{}; + if (values == nullptr || !output_info(index, info) || count != info.bytes) + return false; + const int8_t* source = output_data(index); + for (size_t i = 0; i < count; ++i) { + values[i] = static_cast(source[i] - info.zero_point) * info.scale; + } + return true; +} + +int32_t Int8Interpreter::output_argmax(size_t index) const { + Int8TensorInfo info{}; + if (!output_info(index, info)) + return -1; + const int8_t* source = output_data(index); + int32_t best = 0; + for (size_t i = 1; i < info.bytes; ++i) { + if (source[i] > source[best]) + best = static_cast(i); + } + return best; +} + +int64_t Int8Interpreter::op_time_us(uint32_t op) const { + return loaded_ && op < graph_.header.num_operations ? op_states_[op].last_us + : 0; +} + +bool Int8Interpreter::run_operation(uint32_t index) { + OpState& s = op_states_[index]; + const bool reference = options_.kernel_mode == KernelMode::REFERENCE; + const int8_t* input = activation(s.input0); + int8_t* output = activation(s.output); + switch (s.opcode) { + case OpCode::CONV2D: + if (s.generic) { + run_generic_conv(s, input, output); + return true; + } + if (reference) { + esp_nn_conv_s8_ansi( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.conv, + &s.quant); + } else { + esp_nn_conv_s8( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.conv, + &s.quant); + } + return true; + case OpCode::DEPTHWISE_CONV2D: + if (s.generic) { + run_generic_conv(s, input, output); + return true; + } + if (reference) { + esp_nn_depthwise_conv_s8_ansi( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.dw, + &s.quant); + } else { + esp_nn_depthwise_conv_s8( + &s.input_dims, + input, + &s.filter_dims, + s.weights, + s.bias, + &s.output_dims, + output, + &s.dw, + &s.quant); + } + return true; + case OpCode::RELU6: + for (uint32_t i = 0; i < s.elements; ++i) { + int32_t value = input[i]; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + output[i] = static_cast(value); + } + return true; + case OpCode::ADD: { + input = operand(s.input0); + const int8_t* rhs = operand(s.input1); + if (reference) { + esp_nn_add_elementwise_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + s.add.input1_mult, + s.add.input2_mult, + s.add.input1_shift, + s.add.input2_shift, + s.add.left_shift, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } else { + esp_nn_add_elementwise_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + s.add.input1_mult, + s.add.input2_mult, + s.add.input1_shift, + s.add.input2_shift, + s.add.left_shift, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } + return true; + } + case OpCode::AVGPOOL2D: + if (reference) { + esp_nn_mean_nhwc_s8_ansi( + input, + output, + s.input_dims.height, + s.input_dims.width, + s.input_dims.channels, + s.input_zero_point, + s.output_zero_point, + s.value0, + s.value1); + } else { + esp_nn_mean_nhwc_s8( + input, + output, + s.input_dims.height, + s.input_dims.width, + s.input_dims.channels, + s.input_zero_point, + s.output_zero_point, + s.value0, + s.value1); + } + return true; + case OpCode::FLATTEN: + if (s.transpose) { + const int32_t plane = s.input_dims.height * s.input_dims.width; + const int32_t channels = s.input_dims.channels; + for (int32_t c = 0; c < channels; ++c) { + for (int32_t p = 0; p < plane; ++p) { + output[c * plane + p] = input[p * channels + c]; + } + } + } else { + std::memcpy(output, input, s.elements); + } + return true; + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: { + const uint16_t in_w = static_cast(s.input_dims.width); + const uint16_t in_h = static_cast(s.input_dims.height); + const uint16_t out_w = static_cast(s.output_dims.width); + const uint16_t out_h = static_cast(s.output_dims.height); + const uint16_t channels = static_cast(s.input_dims.channels); + const uint16_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], + sw = s.pool[3]; + const uint16_t ph = s.pool[4], pw = s.pool[5]; + if (s.opcode == OpCode::MAX_POOL2D) { + if (reference) { + esp_nn_max_pool_s8_ansi( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } else { + esp_nn_max_pool_s8( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } + } else if (reference) { + esp_nn_avg_pool_s8_ansi( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } else { + esp_nn_avg_pool_s8( + input, + in_w, + in_h, + output, + out_w, + out_h, + sw, + sh, + kw, + kh, + pw, + ph, + s.value0, + s.value1, + channels); + } + return true; + } + case OpCode::LUT: { + const uint8_t* table = reinterpret_cast(s.weights) + 128; + const uint32_t words = s.elements / 4; + const uint32_t* source = reinterpret_cast(input); + uint32_t* destination = reinterpret_cast(output); + for (uint32_t w = 0; w < words; ++w) { + const uint32_t packed = source[w]; + destination[w] = + static_cast(table[static_cast(packed)]) | + (static_cast(table[static_cast(packed >> 8)]) + << 8) | + (static_cast(table[static_cast(packed >> 16)]) + << 16) | + (static_cast(table[static_cast(packed >> 24)]) + << 24); + } + for (uint32_t i = words * 4; i < s.elements; ++i) { + output[i] = static_cast(table[input[i]]); + } + return true; + } + case OpCode::MUL: { + input = operand(s.input0); + const int8_t* rhs = operand(s.input1); + if (s.transpose) { + if (reference) { + esp_nn_mul_broadcast_channel_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + s.rows, + s.cols); + } else { + esp_nn_mul_broadcast_channel_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + s.rows, + s.cols); + } + } else if (reference) { + esp_nn_mul_elementwise_s8_ansi( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } else { + esp_nn_mul_elementwise_s8( + input, + rhs, + -s.input_zero_point, + -s.value0, + output, + s.output_zero_point, + s.add.output_mult, + s.add.output_shift, + s.add.activation_min, + s.add.activation_max, + static_cast(s.elements)); + } + return true; + } + case OpCode::SOFTMAX: { + const uint32_t* table = reinterpret_cast(s.multipliers); + for (int32_t r = 0; r < s.rows; ++r) { + const int8_t* row = input + r * s.cols; + int8_t* destination = output + r * s.cols; + int32_t maximum = row[0]; + for (int32_t c = 1; c < s.cols; ++c) { + if (row[c] > maximum) + maximum = row[c]; + } + uint64_t total = 0; + for (int32_t c = 0; c < s.cols; ++c) + total += table[maximum - row[c]]; + for (int32_t c = 0; c < s.cols; ++c) { + const uint64_t numerator = 512ULL * table[maximum - row[c]] + total; + const int32_t value = + static_cast(numerator / (2ULL * total)) + + SOFTMAX_OUTPUT_ZERO_POINT; + destination[c] = static_cast(value > 127 ? 127 : value); + } + } + return true; + } + case OpCode::CONCAT: { + int32_t offset = 0; + for (uint8_t k = 0; k < s.input_count; ++k) { + const int32_t channels = tensors_[s.inputs[k]].shape[1]; + const int8_t* source = activation(s.inputs[k]); + for (int32_t r = 0; r < s.rows; ++r) { + std::memcpy( + output + r * s.cols + offset, source + r * channels, channels); + } + offset += channels; + } + return true; + } + case OpCode::LINEAR: + for (int32_t b = 0; b < s.batches; ++b) { + const int8_t* row = input + static_cast(b) * s.cols; + int8_t* destination = output + static_cast(b) * s.rows; + if (reference) { + esp_nn_fully_connected_per_ch_s8_ansi( + row, + -s.input_zero_point, + static_cast(s.cols), + s.weights, + 0, + s.bias, + destination, + static_cast(s.rows), + s.output_zero_point, + s.shifts, + s.multipliers, + s.value0, + s.value1); + } else { + esp_nn_fully_connected_per_ch_s8( + row, + -s.input_zero_point, + static_cast(s.cols), + s.weights, + 0, + s.bias, + destination, + static_cast(s.rows), + s.output_zero_point, + s.shifts, + s.multipliers, + s.value0, + s.value1); + } + } + return true; + case OpCode::COPY: + run_copy(s, output); + return true; + case OpCode::UPSAMPLE: + run_upsample(s, input, output); + return true; + case OpCode::TRANSPOSE_CONV2D: + run_transposed_conv(s, input, output); + return true; + default: + return false; + } +} + +void Int8Interpreter::run_dilated_by_phases( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const bool reference = options_.kernel_mode == KernelMode::REFERENCE; + const int8_t pad_value = clamp_to_int8(s.input_zero_point); + for (int32_t qy = 0; qy < dh; ++qy) { + const int32_t rows = (out_h - qy + dh - 1) / dh; + if (rows <= 0) + continue; + const int32_t ty = qy - ph; + const int32_t ry = ((ty % dh) + dh) % dh; + const int32_t by = (ty - ry) / dh; + for (int32_t qx = 0; qx < dw; ++qx) { + const int32_t cols = (out_w - qx + dw - 1) / dw; + if (cols <= 0) + continue; + const int32_t tx = qx - pw; + const int32_t rx = ((tx % dw) + dw) % dw; + const int32_t bx = (tx - rx) / dw; + const int32_t tile_h = rows + kh - 1; + const int32_t tile_w = cols + kw - 1; + int8_t* tile = phase_buffer_; + int8_t* result = + phase_buffer_ + static_cast(tile_h) * tile_w * in_c; + for (int32_t i = 0; i < tile_h; ++i) { + const int32_t iy = ry + dh * (by + i); + int8_t* row = tile + static_cast(i) * tile_w * in_c; + for (int32_t j = 0; j < tile_w; ++j) { + const int32_t ix = rx + dw * (bx + j); + int8_t* pixel = row + static_cast(j) * in_c; + if (iy < 0 || iy >= in_h || ix < 0 || ix >= in_w) { + std::memset(pixel, pad_value, in_c); + } else { + std::memcpy( + pixel, + input + (static_cast(iy) * in_w + ix) * in_c, + in_c); + } + } + } + data_dims_t tile_dims{tile_w, tile_h, in_c, 1}; + data_dims_t result_dims{cols, rows, out_c, 1}; + if (s.opcode == OpCode::CONV2D) { + if (reference) { + esp_nn_conv_s8_ansi( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.conv, + &s.quant); + } else { + esp_nn_conv_s8( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.conv, + &s.quant); + } + } else if (reference) { + esp_nn_depthwise_conv_s8_ansi( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.dw, + &s.quant); + } else { + esp_nn_depthwise_conv_s8( + &tile_dims, + tile, + &s.filter_dims, + s.weights, + s.bias, + &result_dims, + result, + &s.dw, + &s.quant); + } + for (int32_t i = 0; i < rows; ++i) { + const int32_t oy = qy + dh * i; + for (int32_t j = 0; j < cols; ++j) { + const int32_t ox = qx + dw * j; + std::memcpy( + output + (static_cast(oy) * out_w + ox) * out_c, + result + (static_cast(i) * cols + j) * out_c, + out_c); + } + } + } + } +} + +void Int8Interpreter::run_generic_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const { + if (s.transpose) { + run_dilated_by_phases(s, input, output); + return; + } + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], sw = s.pool[3]; + const int32_t ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const bool depthwise = s.opcode == OpCode::DEPTHWISE_CONV2D; + const int32_t groups = depthwise ? in_c : s.rows; + const int32_t group_in = depthwise ? 1 : in_c / groups; + const int32_t group_out = out_c / groups; + const int32_t in_offset = -s.input_zero_point; + for (int32_t oy = 0; oy < out_h; ++oy) { + for (int32_t ox = 0; ox < out_w; ++ox) { + int8_t* destination = + output + (static_cast(oy) * out_w + ox) * out_c; + for (int32_t oc = 0; oc < out_c; ++oc) { + const int32_t g = oc / group_out; + int32_t acc = s.bias != nullptr ? s.bias[oc] : 0; + for (int32_t ky = 0; ky < kh; ++ky) { + const int32_t iy = oy * sh - ph + ky * dh; + if (iy < 0 || iy >= in_h) + continue; + for (int32_t kx = 0; kx < kw; ++kx) { + const int32_t ix = ox * sw - pw + kx * dw; + if (ix < 0 || ix >= in_w) + continue; + const int8_t* pixel = + input + (static_cast(iy) * in_w + ix) * in_c; + if (depthwise) { + const int32_t ic = oc / (out_c / in_c); + acc += (pixel[ic] + in_offset) * + s.weights[(static_cast(ky) * kw + kx) * out_c + oc]; + } else { + const int8_t* filter = s.weights + + ((static_cast(oc) * kh + ky) * kw + kx) * group_in; + const int8_t* source = pixel + g * group_in; + for (int32_t ic = 0; ic < group_in; ++ic) { + acc += (source[ic] + in_offset) * filter[ic]; + } + } + } + } + int32_t value = requantize(acc, s.multipliers[oc], s.shifts[oc]) + + s.output_zero_point; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + destination[oc] = static_cast(value); + } + } + } +} + +void Int8Interpreter::run_copy(const OpState& s, int8_t* output) const { + const uint8_t* p = s.table; + const int32_t segments = read_le_i32(p); + const int32_t flags = read_le_i32(p + 4); + if ((flags & COPY_FLAG_FILL) != 0) { + std::memset( + output, read_le_i32(p + 8), tensor_data_size(tensors_[s.output])); + } + for (int32_t k = 0; k < segments; ++k) { + const uint8_t* q = + p + COPY_HEADER_SIZE + static_cast(k) * COPY_SEGMENT_SIZE; + const int8_t* source = operand(s.inputs[read_le_i32(q)]); + const int32_t rank = read_le_i32(q + 4); + const int32_t in_base = read_le_i32(q + 8); + const int32_t out_base = read_le_i32(q + 12); + int32_t dims[COPY_MAX_RANK]; + int32_t in_strides[COPY_MAX_RANK]; + int32_t out_strides[COPY_MAX_RANK]; + for (int32_t d = 0; d < static_cast(COPY_MAX_RANK); ++d) { + const int32_t from = d - (static_cast(COPY_MAX_RANK) - rank); + dims[d] = from >= 0 ? read_le_i32(q + 16 + 4 * from) : 1; + in_strides[d] = from >= 0 ? read_le_i32(q + 40 + 4 * from) : 0; + out_strides[d] = from >= 0 ? read_le_i32(q + 64 + 4 * from) : 0; + } + const bool requant = read_le_i32(q + 88) != 0; + const int32_t multiplier = read_le_i32(q + 92); + const int32_t shift = read_le_i32(q + 96); + const int32_t in_zp = read_le_i32(q + 100); + const int32_t out_zp = read_le_i32(q + 104); + const int32_t inner = dims[5]; + const bool contiguous = + in_strides[5] == 1 && out_strides[5] == 1 && !requant; + for (int32_t a = 0; a < dims[0]; ++a) + for (int32_t b = 0; b < dims[1]; ++b) + for (int32_t c = 0; c < dims[2]; ++c) + for (int32_t d = 0; d < dims[3]; ++d) + for (int32_t e = 0; e < dims[4]; ++e) { + const int32_t in_index = in_base + a * in_strides[0] + + b * in_strides[1] + c * in_strides[2] + d * in_strides[3] + + e * in_strides[4]; + const int32_t out_index = out_base + a * out_strides[0] + + b * out_strides[1] + c * out_strides[2] + d * out_strides[3] + + e * out_strides[4]; + if (contiguous) { + std::memcpy(output + out_index, source + in_index, inner); + continue; + } + const int8_t* from = source + in_index; + int8_t* to = output + out_index; + for (int32_t f = 0; f < inner; ++f) { + int32_t value = from[f * in_strides[5]]; + if (requant) + value = clamp_to_int8( + requantize(value - in_zp, multiplier, shift) + out_zp); + to[f * out_strides[5]] = static_cast(value); + } + } + } +} + +void Int8Interpreter::run_upsample( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_w = s.input_dims.width; + const int32_t channels = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const uint8_t* y_table = s.table + UPSAMPLE_HEADER_SIZE; + const uint8_t* x_table = y_table + 12U * static_cast(out_h); + const int32_t one = 1 << UPSAMPLE_WEIGHT_BITS; + const int32_t half = 1 << (2 * UPSAMPLE_WEIGHT_BITS - 1); + const bool nearest = s.value0 == UPSAMPLE_MODE_NEAREST; + for (int32_t oy = 0; oy < out_h; ++oy) { + const int32_t y0 = read_le_i32(y_table + 12 * oy); + const int32_t y1 = read_le_i32(y_table + 12 * oy + 4); + const int32_t wy = read_le_i32(y_table + 12 * oy + 8); + const int8_t* row0 = input + static_cast(y0) * in_w * channels; + const int8_t* row1 = input + static_cast(y1) * in_w * channels; + int8_t* destination = output + static_cast(oy) * out_w * channels; + for (int32_t ox = 0; ox < out_w; ++ox) { + const int32_t x0 = read_le_i32(x_table + 12 * ox); + const int32_t x1 = read_le_i32(x_table + 12 * ox + 4); + const int32_t wx = read_le_i32(x_table + 12 * ox + 8); + int8_t* pixel = destination + static_cast(ox) * channels; + if (nearest) { + std::memcpy(pixel, row0 + static_cast(x0) * channels, channels); + continue; + } + const int8_t* a = row0 + static_cast(x0) * channels; + const int8_t* b = row0 + static_cast(x1) * channels; + const int8_t* c = row1 + static_cast(x0) * channels; + const int8_t* d = row1 + static_cast(x1) * channels; + for (int32_t ch = 0; ch < channels; ++ch) { + const int32_t top = a[ch] * (one - wx) + b[ch] * wx; + const int32_t bottom = c[ch] * (one - wx) + d[ch] * wx; + const int32_t total = top * (one - wy) + bottom * wy + half; + pixel[ch] = clamp_to_int8(total >> (2 * UPSAMPLE_WEIGHT_BITS)); + } + } + } +} + +void Int8Interpreter::run_transposed_conv( + const OpState& s, + const int8_t* input, + int8_t* output) const { + const int32_t in_h = s.input_dims.height; + const int32_t in_w = s.input_dims.width; + const int32_t in_c = s.input_dims.channels; + const int32_t out_h = s.output_dims.height; + const int32_t out_w = s.output_dims.width; + const int32_t out_c = s.output_dims.channels; + const int32_t kh = s.pool[0], kw = s.pool[1], sh = s.pool[2], sw = s.pool[3]; + const int32_t ph = s.pool[4], pw = s.pool[5]; + const int32_t dh = s.dilation_h, dw = s.dilation_w; + const int32_t in_offset = -s.input_zero_point; + for (int32_t oy = 0; oy < out_h; ++oy) { + for (int32_t ox = 0; ox < out_w; ++ox) { + int8_t* destination = + output + (static_cast(oy) * out_w + ox) * out_c; + for (int32_t oc = 0; oc < out_c; ++oc) { + int32_t acc = s.bias[oc]; + for (int32_t ky = 0; ky < kh; ++ky) { + const int32_t ty = oy + ph - ky * dh; + if (ty < 0 || ty % sh != 0) + continue; + const int32_t iy = ty / sh; + if (iy >= in_h) + continue; + for (int32_t kx = 0; kx < kw; ++kx) { + const int32_t tx = ox + pw - kx * dw; + if (tx < 0 || tx % sw != 0) + continue; + const int32_t ix = tx / sw; + if (ix >= in_w) + continue; + const int8_t* pixel = + input + (static_cast(iy) * in_w + ix) * in_c; + const int8_t* filter = s.weights + + ((static_cast(oc) * kh + ky) * kw + kx) * in_c; + for (int32_t ic = 0; ic < in_c; ++ic) + acc += (pixel[ic] + in_offset) * filter[ic]; + } + } + int32_t value = requantize(acc, s.multipliers[oc], s.shifts[oc]) + + s.output_zero_point; + if (value < s.value0) + value = s.value0; + if (value > s.value1) + value = s.value1; + destination[oc] = static_cast(value); + } + } + } +} + +bool Int8Interpreter::invoke() { + if (!loaded_) + return false; + InvokeLock lock; + if (scratch_ != nullptr) { + esp_nn_set_conv_scratch_buf(scratch_); + esp_nn_set_depthwise_conv_scratch_buf(scratch_); + esp_nn_set_mean_scratch_buf(scratch_); + } + int64_t compute_us = 0; + int64_t last_yield = esp_timer_get_time(); + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + const int64_t started = esp_timer_get_time(); + if (!run_operation(i)) { + std::printf( + "ERROR: operation %u (%s) failed\n", + static_cast(i), + opcode_name(op_states_[i].opcode)); + return false; + } + const int64_t finished = esp_timer_get_time(); + OpState& state = op_states_[i]; + state.last_us = finished - started; + state.total_us += state.last_us; + ++state.calls; + compute_us += state.last_us; + if (finished - last_yield > YIELD_INTERVAL_US) { + vTaskDelay(1); + last_yield = esp_timer_get_time(); + } + } + last_invoke_us_ = compute_us; + return true; +} + +void Int8Interpreter::print_summary() const { + if (!loaded_) + return; + std::printf( + "Int8 model: %u tensors, %u operations, %u constant bytes\n", + static_cast(graph_.header.num_tensors), + static_cast(graph_.header.num_operations), + static_cast(graph_.header.constant_size)); + std::printf( + "Activation arena: %u bytes in %s\n", + static_cast(arena_size_), + arena_internal_ ? "internal RAM" : "PSRAM"); + std::printf( + "ESP-NN scratch: %u bytes; requant tables: %u bytes; constants %s\n", + static_cast(scratch_size_), + static_cast(params_size_), + blob_copy_ != nullptr ? "copied to RAM" : "read in place from flash"); + for (size_t i = 0; i < input_count_; ++i) { + Int8TensorInfo info{}; + input_info(i, info); + std::printf( + "Input %u: tensor %u HWC=%ldx%ldx%ld scale=%.9g zero_point=%ld\n", + static_cast(i), + static_cast(info.tensor_id), + static_cast(info.height), + static_cast(info.width), + static_cast(info.channels), + info.scale, + static_cast(info.zero_point)); + } + for (size_t i = 0; i < output_count_; ++i) { + Int8TensorInfo info{}; + output_info(i, info); + std::printf( + "Output %u: tensor %u elements=%u scale=%.9g zero_point=%ld\n", + static_cast(i), + static_cast(info.tensor_id), + static_cast(info.bytes), + info.scale, + static_cast(info.zero_point)); + } +} + +void Int8Interpreter::print_profile() const { + if (!loaded_) + return; + int64_t per_type[OPCODE_COUNT]{}; + uint32_t count_per_type[OPCODE_COUNT]{}; + for (uint32_t i = 0; i < graph_.header.num_operations; ++i) { + const OpState& s = op_states_[i]; + const unsigned type = static_cast(s.opcode); + if (type < OPCODE_COUNT) { + per_type[type] += s.last_us; + ++count_per_type[type]; + } + std::printf( + " op %2u %-16s out=%ldx%ldx%ld %8lld us\n", + static_cast(i), + opcode_name(s.opcode), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[2] : 1), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[3] : 1), + static_cast( + tensors_[s.output].rank == 4 ? tensors_[s.output].shape[1] + : static_cast(s.elements)), + static_cast(s.last_us)); + } + for (unsigned t = 0; t < OPCODE_COUNT; ++t) { + if (count_per_type[t] == 0) + continue; + std::printf( + " %-16s x%-3u %8lld us\n", + opcode_name(static_cast(t)), + static_cast(count_per_type[t]), + static_cast(per_type[t])); + } +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/espnn_runtime/src/xts3_graph.cpp b/backends/espressif/espnn_runtime/src/xts3_graph.cpp new file mode 100644 index 00000000000..af6b6c38453 --- /dev/null +++ b/backends/espressif/espnn_runtime/src/xts3_graph.cpp @@ -0,0 +1,1064 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +namespace executorch { +namespace backends { +namespace espressif { +namespace { + +uint16_t read_u16(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8); +} + +uint32_t read_u32(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +int32_t read_i32(const uint8_t* p) { + return static_cast(read_u32(p)); +} + +bool range_valid(size_t offset, size_t length, size_t total) { + return offset <= total && length <= total - offset; +} + +bool valid_dtype(uint8_t dtype) { + return dtype >= static_cast(DataType::FLOAT32) && + dtype <= static_cast(DataType::INT32); +} + +bool valid_opcode(uint8_t opcode, uint32_t version) { + if (version == GRAPH_ESP_NN_INT8_IO_VERSION) { + return opcode <= static_cast(OpCode::TRANSPOSE_CONV2D); + } + return opcode <= static_cast(OpCode::SOFTMAX); +} + +bool valid_tensor_reference(const GraphView& graph, uint32_t id) { + return id < graph.header.num_tensors; +} + +bool same_shape(const TensorRecord& a, const TensorRecord& b) { + return a.rank == b.rank && + std::memcmp(a.shape, b.shape, sizeof(a.shape)) == 0; +} + +bool same_quantization(const TensorRecord& a, const TensorRecord& b) { + return a.quant_scale == b.quant_scale && + a.quant_zero_point == b.quant_zero_point; +} + +bool valid_requantization(int32_t multiplier, int32_t shift) { + return multiplier > 0 && shift >= -31 && shift <= 30; +} + +bool valid_activation_range(int32_t low, int32_t high) { + return low >= -128 && high <= 127 && low <= high; +} + +bool is_int8_activation(const TensorRecord& tensor) { + return tensor.dtype == static_cast(DataType::INT8) && + (tensor.flags & TENSOR_FLAG_CONSTANT) == 0 && + (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; +} + +bool is_int8_value(const TensorRecord& tensor) { + return tensor.dtype == static_cast(DataType::INT8) && + (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; +} + +bool valid_layout(const TensorRecord& tensor) { + return tensor.rank >= 1 && tensor.rank <= 4 && + (tensor.rank != 4 || tensor.shape[0] == 1); +} + +bool is_constant_of(const TensorRecord& tensor, DataType dtype) { + return tensor.dtype == static_cast(dtype) && + (tensor.flags & TENSOR_FLAG_CONSTANT) != 0 && + (tensor.flags & TENSOR_FLAG_QUANTIZED) == 0; +} + +bool validate_int8_conv( + const GraphView& graph, + const OperationRecord& op, + bool depthwise) { + if (op.num_inputs != 3 || op.num_outputs != 1) + return false; + TensorRecord input{}; + TensorRecord weight{}; + TensorRecord bias{}; + TensorRecord output{}; + if (!read_tensor(graph, op.inputs[0], input) || + !read_tensor(graph, op.inputs[1], weight) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], output)) + return false; + if (!is_int8_activation(input) || !is_int8_activation(output) || + !is_constant_of(weight, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || input.rank != 4 || + weight.rank != 4 || bias.rank != 1 || output.rank != 4 || + input.shape[0] != 1 || output.shape[0] != 1 || + weight.shape[0] != output.shape[1] || bias.shape[0] != output.shape[1]) + return false; + + const uint32_t channels = static_cast(output.shape[1]); + Int8ConvParams params{}; + if (!read_int8_conv_params(graph, op, channels, params)) + return false; + if (params.dilation_h <= 0 || params.dilation_w <= 0) + return false; + if (graph.header.version == GRAPH_ESP_NN_INT8_VERSION && + (params.dilation_h != 1 || params.dilation_w != 1)) + return false; + const int32_t effective_h = params.dilation_h * (params.kernel_h - 1) + 1; + const int32_t effective_w = params.dilation_w * (params.kernel_w - 1) + 1; + + if (depthwise) { + if (params.groups != input.shape[1] || weight.shape[1] != 1 || + output.shape[1] % input.shape[1] != 0) + return false; + } else if ( + params.groups <= 0 || + static_cast(weight.shape[1]) * params.groups != input.shape[1] || + output.shape[1] % params.groups != 0) { + return false; + } + if (params.kernel_h != weight.shape[2] || + params.kernel_w != weight.shape[3] || params.stride_h <= 0 || + params.stride_w <= 0 || params.pad_h >= effective_h || + params.pad_w >= effective_w || + !valid_activation_range(params.activation_min, params.activation_max)) + return false; + + const int64_t expected_h = + (static_cast(input.shape[2]) + 2 * params.pad_h - effective_h) / + params.stride_h + + 1; + const int64_t expected_w = + (static_cast(input.shape[3]) + 2 * params.pad_w - effective_w) / + params.stride_w + + 1; + if (output.shape[2] != expected_h || output.shape[3] != expected_w) + return false; + if (input.shape[1] > 65535 || input.shape[2] > 65535 || + input.shape[3] > 65535 || output.shape[1] > 65535) + return false; + for (uint32_t i = 0; i < channels; ++i) { + if (!valid_requantization( + read_i32(params.multipliers + i * 4), + read_i32(params.shifts + i * 4))) + return false; + } + return true; +} + +bool copy_extent_valid( + int32_t rank, + const int32_t* dims, + const int32_t* strides, + int32_t offset, + size_t limit) { + if (limit == 0 || limit > static_cast(INT32_MAX)) + return false; + const int64_t bound = static_cast(limit); + int64_t low = offset; + int64_t high = offset; + for (int32_t d = 0; d < rank; ++d) { + if (dims[d] <= 0) + return false; + const int64_t span = static_cast(dims[d] - 1) * strides[d]; + if (span <= -bound || span >= bound) + return false; + if (span < 0) + low += span; + else + high += span; + } + return low >= 0 && high < bound; +} + +bool validate_copy(const GraphView& graph, const OperationRecord& op) { + TensorRecord out{}; + if (op.num_outputs != 1 || op.parameter_size < COPY_HEADER_SIZE || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(out) || + !valid_layout(out)) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + const int32_t segments = read_i32(p); + const int32_t flags = read_i32(p + 4); + const int32_t fill = read_i32(p + 8); + if (segments <= 0 || (flags & ~COPY_FLAG_FILL) != 0 || fill < -128 || + fill > 127 || + static_cast(segments) > + (op.parameter_size - COPY_HEADER_SIZE) / COPY_SEGMENT_SIZE || + op.parameter_size != + COPY_HEADER_SIZE + + static_cast(segments) * COPY_SEGMENT_SIZE) { + return false; + } + TensorRecord inputs[6]{}; + for (uint8_t j = 0; j < op.num_inputs; ++j) { + if (!read_tensor(graph, op.inputs[j], inputs[j]) || + !is_int8_value(inputs[j]) || !valid_layout(inputs[j])) + return false; + } + const size_t out_count = tensor_element_count(out); + for (int32_t k = 0; k < segments; ++k) { + const uint8_t* q = + p + COPY_HEADER_SIZE + static_cast(k) * COPY_SEGMENT_SIZE; + const int32_t index = read_i32(q); + const int32_t rank = read_i32(q + 4); + const int32_t in_offset = read_i32(q + 8); + const int32_t out_offset = read_i32(q + 12); + int32_t dims[COPY_MAX_RANK]; + int32_t in_strides[COPY_MAX_RANK]; + int32_t out_strides[COPY_MAX_RANK]; + for (size_t d = 0; d < COPY_MAX_RANK; ++d) { + dims[d] = read_i32(q + 16 + 4 * d); + in_strides[d] = read_i32(q + 40 + 4 * d); + out_strides[d] = read_i32(q + 64 + 4 * d); + } + const int32_t requant = read_i32(q + 88); + if (index < 0 || index >= op.num_inputs || rank <= 0 || + rank > static_cast(COPY_MAX_RANK) || + (requant != 0 && requant != 1)) + return false; + const TensorRecord& source = inputs[index]; + if (!copy_extent_valid( + rank, dims, in_strides, in_offset, tensor_element_count(source)) || + !copy_extent_valid(rank, dims, out_strides, out_offset, out_count)) + return false; + if (requant != 0) { + if (!valid_requantization(read_i32(q + 92), read_i32(q + 96)) || + read_i32(q + 100) != source.quant_zero_point || + read_i32(q + 104) != out.quant_zero_point) + return false; + } else if (!same_quantization(source, out)) { + return false; + } + } + return true; +} + +bool validate_upsample(const GraphView& graph, const OperationRecord& op) { + TensorRecord a{}; + TensorRecord out{}; + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size < UPSAMPLE_HEADER_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_quantization(a, out) || a.rank != 4 || + out.rank != 4 || a.shape[0] != 1 || out.shape[0] != 1 || + a.shape[1] != out.shape[1]) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + const int32_t mode = read_i32(p); + const int32_t in_h = read_i32(p + 4); + const int32_t in_w = read_i32(p + 8); + const int32_t out_h = read_i32(p + 12); + const int32_t out_w = read_i32(p + 16); + if ((mode != UPSAMPLE_MODE_NEAREST && mode != UPSAMPLE_MODE_BILINEAR) || + in_h != a.shape[2] || in_w != a.shape[3] || out_h != out.shape[2] || + out_w != out.shape[3] || out_h <= 0 || out_w <= 0 || + out_h > UPSAMPLE_MAX_EXTENT || out_w > UPSAMPLE_MAX_EXTENT || + static_cast(op.parameter_size) != + UPSAMPLE_HEADER_SIZE + + 12ULL * (static_cast(out_h) + out_w)) { + return false; + } + const int32_t one = 1 << UPSAMPLE_WEIGHT_BITS; + const uint8_t* table = p + UPSAMPLE_HEADER_SIZE; + for (int32_t i = 0; i < out_h + out_w; ++i) { + const int32_t limit = i < out_h ? in_h : in_w; + const int32_t i0 = read_i32(table + 12 * i); + const int32_t i1 = read_i32(table + 12 * i + 4); + const int32_t weight = read_i32(table + 12 * i + 8); + if (i0 < 0 || i0 >= limit || i1 < 0 || i1 >= limit || weight < 0 || + weight > one) + return false; + } + return true; +} + +bool validate_transposed_conv( + const GraphView& graph, + const OperationRecord& op) { + TensorRecord input{}; + TensorRecord weight{}; + TensorRecord bias{}; + TensorRecord output{}; + if (op.num_inputs != 3 || op.num_outputs != 1 || + !read_tensor(graph, op.inputs[0], input) || + !read_tensor(graph, op.inputs[1], weight) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], output) || + !is_int8_activation(input) || !is_int8_activation(output) || + !is_constant_of(weight, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || input.rank != 4 || + output.rank != 4 || weight.rank != 4 || bias.rank != 1 || + input.shape[0] != 1 || output.shape[0] != 1 || + weight.shape[0] != output.shape[1] || weight.shape[3] != input.shape[1] || + bias.shape[0] != output.shape[1]) + return false; + const size_t channels = static_cast(output.shape[1]); + if (op.parameter_size != TRANSPOSE_CONV_PARAMS_SIZE + channels * 8U) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + int32_t f[13]; + for (int k = 0; k < 13; ++k) + f[k] = read_i32(p + 4 * k); + const int32_t kh = f[0], kw = f[1], sh = f[2], sw = f[3], ph = f[4], + pw = f[5]; + const int32_t oph = f[6], opw = f[7], dh = f[8], dw = f[9], groups = f[10]; + if (kh != weight.shape[1] || kw != weight.shape[2] || sh <= 0 || sw <= 0 || + ph < 0 || pw < 0 || oph < 0 || opw < 0 || dh <= 0 || dw <= 0 || + groups != 1 || !valid_activation_range(f[11], f[12])) + return false; + const int64_t expected_h = static_cast(input.shape[2] - 1) * sh - + 2 * ph + static_cast(dh) * (kh - 1) + oph + 1; + const int64_t expected_w = static_cast(input.shape[3] - 1) * sw - + 2 * pw + static_cast(dw) * (kw - 1) + opw + 1; + if (output.shape[2] != expected_h || output.shape[3] != expected_w) + return false; + const uint8_t* multipliers = p + TRANSPOSE_CONV_PARAMS_SIZE; + const uint8_t* shifts = multipliers + channels * 4U; + for (size_t c = 0; c < channels; ++c) { + if (!valid_requantization( + read_i32(multipliers + 4 * c), read_i32(shifts + 4 * c))) + return false; + } + return true; +} + +bool validate_int8_operation( + const GraphView& graph, + const OperationRecord& op) { + TensorRecord a{}; + TensorRecord b{}; + TensorRecord out{}; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + switch (static_cast(op.opcode)) { + case OpCode::CONV2D: + return validate_int8_conv(graph, op, false); + case OpCode::DEPTHWISE_CONV2D: + return validate_int8_conv(graph, op, true); + case OpCode::RELU6: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_PAIR_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_shape(a, out) && + same_quantization(a, out) && + valid_activation_range(read_i32(p), read_i32(p + 4)); + case OpCode::ADD: { + Int8AddParams params{}; + return op.num_inputs == 2 && op.num_outputs == 1 && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.inputs[1], b) && + read_tensor(graph, op.outputs[0], out) && is_int8_value(a) && + is_int8_value(b) && is_int8_activation(out) && same_shape(a, b) && + same_shape(a, out) && read_int8_add_params(graph, op, params) && + params.input1_mult > 0 && params.input2_mult > 0 && + params.output_mult > 0 && params.input1_shift <= 0 && + params.input1_shift >= -31 && params.input2_shift <= 0 && + params.input2_shift >= -31 && params.output_shift <= 0 && + params.output_shift >= -31 && params.left_shift >= 0 && + params.left_shift <= 22 && + valid_activation_range(params.activation_min, params.activation_max); + } + case OpCode::AVGPOOL2D: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_PAIR_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && a.rank == 4 && out.rank == 4 && + a.shape[0] == 1 && out.shape[0] == 1 && a.shape[1] == out.shape[1] && + out.shape[2] == 1 && out.shape[3] == 1 && + static_cast(a.shape[2]) * a.shape[3] * 255 < INT32_MAX && + valid_requantization(read_i32(p), read_i32(p + 4)); + case OpCode::FLATTEN: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == 0 && read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_quantization(a, out) && + out.rank == 2 && out.shape[0] == 1 && + (a.rank != 4 || a.shape[0] == 1) && + tensor_element_count(a) == tensor_element_count(out); + case OpCode::MAX_POOL2D: + case OpCode::AVG_POOL2D: { + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size != INT8_POOL_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_quantization(a, out) || + a.rank != 4 || out.rank != 4 || a.shape[0] != 1 || + out.shape[0] != 1 || a.shape[1] != out.shape[1] || + a.shape[1] > 65535 || a.shape[2] > 65535 || a.shape[3] > 65535) + return false; + const int32_t kh = p[0]; + const int32_t kw = p[1]; + const int32_t sh = p[2]; + const int32_t sw = p[3]; + const int32_t ph = p[4]; + const int32_t pw = p[5]; + if (kh == 0 || kw == 0 || sh == 0 || sw == 0 || p[6] != 1 || p[7] != 1 || + 2 * ph > kh || 2 * pw > kw || + !valid_activation_range(read_i32(p + 8), read_i32(p + 12))) + return false; + return out.shape[2] == (a.shape[2] + 2 * ph - kh) / sh + 1 && + out.shape[3] == (a.shape[3] + 2 * pw - kw) / sw + 1; + } + case OpCode::LUT: + return op.num_inputs == 1 && op.num_outputs == 1 && + op.parameter_size == INT8_LUT_PARAMS_SIZE && + read_tensor(graph, op.inputs[0], a) && + read_tensor(graph, op.outputs[0], out) && is_int8_activation(a) && + is_int8_activation(out) && same_shape(a, out); + case OpCode::MUL: { + if (op.num_inputs != 2 || op.num_outputs != 1 || + op.parameter_size != INT8_MUL_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.inputs[1], b) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_value(a) || + !is_int8_value(b) || !is_int8_activation(out) || + !same_shape(a, out) || + !valid_requantization(read_i32(p), read_i32(p + 4)) || + !valid_activation_range(read_i32(p + 8), read_i32(p + 12))) + return false; + const int32_t mode = read_i32(p + 16); + if (mode == MUL_MODE_ELEMENTWISE) + return same_shape(a, b); + return mode == MUL_MODE_CHANNEL_BROADCAST && a.rank == 4 && b.rank == 4 && + a.shape[0] == 1 && b.shape[0] == 1 && b.shape[1] == a.shape[1] && + b.shape[2] == 1 && b.shape[3] == 1; + } + case OpCode::SOFTMAX: { + if (op.num_inputs != 1 || op.num_outputs != 1 || + op.parameter_size != INT8_SOFTMAX_PARAMS_SIZE || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !same_shape(a, out) || + (a.rank != 2 && a.rank != 4) || a.shape[0] != 1 || + out.quant_zero_point != SOFTMAX_OUTPUT_ZERO_POINT || + std::fabs(out.quant_scale - 1.0f / 256.0f) > 1e-9f) + return false; + uint32_t previous = read_u32(p); + if (previous == 0 || previous > (1U << 24)) + return false; + for (size_t k = 1; k < 256; ++k) { + const uint32_t value = read_u32(p + 4 * k); + if (value > previous) + return false; + previous = value; + } + return true; + } + case OpCode::CONCAT: { + if (op.num_inputs < 2 || op.num_outputs != 1 || op.parameter_size != 0 || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(out) || + (out.rank != 2 && out.rank != 4) || out.shape[0] != 1) + return false; + int64_t channels = 0; + for (uint8_t j = 0; j < op.num_inputs; ++j) { + if (!read_tensor(graph, op.inputs[j], a) || !is_int8_activation(a) || + !same_quantization(a, out) || a.rank != out.rank || a.shape[0] != 1) + return false; + for (uint8_t d = 2; d < out.rank; ++d) { + if (a.shape[d] != out.shape[d]) + return false; + } + channels += a.shape[1]; + } + return channels == out.shape[1]; + } + case OpCode::LINEAR: { + TensorRecord bias{}; + if (op.num_inputs != 3 || op.num_outputs != 1 || + !read_tensor(graph, op.inputs[0], a) || + !read_tensor(graph, op.inputs[1], b) || + !read_tensor(graph, op.inputs[2], bias) || + !read_tensor(graph, op.outputs[0], out) || !is_int8_activation(a) || + !is_int8_activation(out) || !is_constant_of(b, DataType::INT8) || + !is_constant_of(bias, DataType::INT32) || b.rank != 2 || + bias.rank != 1 || bias.shape[0] != b.shape[0] || b.shape[0] > 65535 || + b.shape[1] > 65535 || a.rank == 4 || out.rank == 4 || + tensor_element_count(a) % static_cast(b.shape[1]) != 0 || + tensor_element_count(out) != + tensor_element_count(a) / static_cast(b.shape[1]) * + static_cast(b.shape[0]) || + op.parameter_size != + INT8_PAIR_PARAMS_SIZE + static_cast(b.shape[0]) * 8U || + !valid_activation_range(read_i32(p), read_i32(p + 4))) + return false; + const uint8_t* multipliers = p + INT8_PAIR_PARAMS_SIZE; + const uint8_t* shifts = + multipliers + static_cast(b.shape[0]) * 4U; + for (int32_t i = 0; i < b.shape[0]; ++i) { + if (!valid_requantization( + read_i32(multipliers + i * 4), read_i32(shifts + i * 4))) + return false; + } + return true; + } + case OpCode::COPY: + return validate_copy(graph, op); + case OpCode::UPSAMPLE: + return validate_upsample(graph, op); + case OpCode::TRANSPOSE_CONV2D: + return validate_transposed_conv(graph, op); + } + return false; +} + +bool validate_int8_dataflow(const GraphView& graph) { + bool produced[MAX_GRAPH_TENSORS]{}; + bool copy_produced[MAX_GRAPH_TENSORS]{}; + uint32_t input_count = 0; + uint32_t output_count = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0) { + produced[i] = true; + ++input_count; + } + if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0) + ++output_count; + } + if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + bool listed_input[MAX_GRAPH_TENSORS]{}; + bool listed_output[MAX_GRAPH_TENSORS]{}; + for (uint32_t i = 0; i < graph.num_inputs; ++i) { + const uint32_t id = graph_input_id(graph, i); + TensorRecord tensor{}; + if (!read_tensor(graph, id, tensor) || + (tensor.flags & TENSOR_FLAG_GRAPH_INPUT) == 0) { + std::printf( + "ERROR: graph IO table lists input %u without the input flag\n", + static_cast(id)); + return false; + } + listed_input[id] = true; + } + for (uint32_t i = 0; i < graph.num_outputs; ++i) { + const uint32_t id = graph_output_id(graph, i); + TensorRecord tensor{}; + if (!read_tensor(graph, id, tensor) || + (tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) == 0) { + std::printf( + "ERROR: graph IO table lists output %u without the output flag\n", + static_cast(id)); + return false; + } + listed_output[id] = true; + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if (((tensor.flags & TENSOR_FLAG_GRAPH_INPUT) != 0 && !listed_input[i]) || + ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && + !listed_output[i])) { + std::printf( + "ERROR: tensor %u is flagged as graph IO but missing from the IO table\n", + static_cast(i)); + return false; + } + } + } + if (input_count == 0 || output_count == 0) { + std::printf( + "ERROR: int8 graph must declare at least one input and one output\n"); + return false; + } + for (uint32_t i = 0; i < graph.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph, i, op); + for (uint8_t j = 0; j < op.num_inputs; ++j) { + TensorRecord tensor{}; + read_tensor(graph, op.inputs[j], tensor); + if ((tensor.flags & TENSOR_FLAG_CONSTANT) == 0 && + !produced[op.inputs[j]]) { + std::printf( + "ERROR: operation %u reads tensor %u before it is produced\n", + static_cast(i), + static_cast(op.inputs[j])); + return false; + } + } + const bool is_copy = static_cast(op.opcode) == OpCode::COPY; + for (uint8_t j = 0; j < op.num_outputs; ++j) { + const uint32_t id = op.outputs[j]; + if (produced[id] && !(is_copy && copy_produced[id])) { + std::printf( + "ERROR: tensor %u has more than one producer\n", + static_cast(id)); + return false; + } + produced[id] = true; + copy_produced[id] = is_copy; + } + } + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT) != 0 && !produced[i]) { + std::printf( + "ERROR: graph output tensor %u is never produced\n", + static_cast(i)); + return false; + } + } + return true; +} + +} // namespace + +uint32_t read_le_u32(const uint8_t* p) { + return read_u32(p); +} + +int32_t read_le_i32(const uint8_t* p) { + return read_i32(p); +} + +bool is_int8_graph_version(uint32_t version) { + return version == GRAPH_ESP_NN_INT8_VERSION || + version == GRAPH_ESP_NN_INT8_IO_VERSION; +} + +namespace { + +uint32_t flagged_tensor(const GraphView& graph, uint8_t flag, uint32_t index) { + uint32_t seen = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & flag) == 0) + continue; + if (seen == index) + return i; + ++seen; + } + return UINT32_MAX; +} + +uint32_t flagged_count(const GraphView& graph, uint8_t flag) { + uint32_t count = 0; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if ((tensor.flags & flag) != 0) + ++count; + } + return count; +} + +} // namespace + +uint32_t graph_input_count(const GraphView& graph) { + if (graph.io_ids != nullptr) + return graph.num_inputs; + return flagged_count(graph, TENSOR_FLAG_GRAPH_INPUT); +} + +uint32_t graph_output_count(const GraphView& graph) { + if (graph.io_ids != nullptr) + return graph.num_outputs; + return flagged_count(graph, TENSOR_FLAG_GRAPH_OUTPUT); +} + +uint32_t graph_input_id(const GraphView& graph, uint32_t index) { + if (graph.io_ids != nullptr) { + return index < graph.num_inputs ? read_u32(graph.io_ids + 4 * index) + : UINT32_MAX; + } + return flagged_tensor(graph, TENSOR_FLAG_GRAPH_INPUT, index); +} + +uint32_t graph_output_id(const GraphView& graph, uint32_t index) { + if (graph.io_ids != nullptr) { + return index < graph.num_outputs + ? read_u32(graph.io_ids + 4 * (graph.num_inputs + index)) + : UINT32_MAX; + } + return flagged_tensor(graph, TENSOR_FLAG_GRAPH_OUTPUT, index); +} + +bool read_int8_conv_params( + const GraphView& graph, + const OperationRecord& op, + uint32_t channels, + Int8ConvParams& params) { + const size_t expected = + INT8_CONV_BASE_PARAMS_SIZE + static_cast(channels) * 8U; + if (op.parameter_size != expected) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + params.kernel_h = p[0]; + params.kernel_w = p[1]; + params.stride_h = p[2]; + params.stride_w = p[3]; + params.pad_h = p[4]; + params.pad_w = p[5]; + params.dilation_h = p[6]; + params.dilation_w = p[7]; + params.groups = read_i32(p + 8); + params.activation_min = read_i32(p + 12); + params.activation_max = read_i32(p + 16); + params.multipliers = p + INT8_CONV_BASE_PARAMS_SIZE; + params.shifts = params.multipliers + static_cast(channels) * 4U; + return true; +} + +bool read_int8_add_params( + const GraphView& graph, + const OperationRecord& op, + Int8AddParams& params) { + if (op.parameter_size != INT8_ADD_PARAMS_SIZE) + return false; + const uint8_t* p = graph.parameter_data + op.parameter_offset; + params.input1_mult = read_i32(p); + params.input1_shift = read_i32(p + 4); + params.input2_mult = read_i32(p + 8); + params.input2_shift = read_i32(p + 12); + params.left_shift = read_i32(p + 16); + params.output_mult = read_i32(p + 20); + params.output_shift = read_i32(p + 24); + params.activation_min = read_i32(p + 28); + params.activation_max = read_i32(p + 32); + return true; +} + +size_t tensor_element_count(const TensorRecord& tensor) { + size_t count = 1; + for (uint8_t i = 0; i < tensor.rank; ++i) { + if (tensor.shape[i] <= 0 || + static_cast(tensor.shape[i]) > SIZE_MAX / count) { + return 0; + } + count *= static_cast(tensor.shape[i]); + } + return count; +} + +size_t tensor_data_size(const TensorRecord& tensor) { + const size_t count = tensor_element_count(tensor); + size_t element_size = 0; + switch (static_cast(tensor.dtype)) { + case DataType::FLOAT32: + case DataType::INT32: + element_size = 4; + break; + case DataType::INT8: + case DataType::UINT8: + element_size = 1; + break; + default: + return 0; + } + if (count == 0 || count > SIZE_MAX / element_size) { + return 0; + } + return count * element_size; +} + +bool read_tensor(const GraphView& graph, uint32_t index, TensorRecord& tensor) { + if (index >= graph.header.num_tensors) + return false; + const uint8_t* p = graph.data + graph.tensor_table_offset + + static_cast(index) * graph.tensor_record_size; + tensor.tensor_id = read_u32(p); + tensor.constant_offset = read_u32(p + 4); + tensor.dtype = p[8]; + tensor.flags = p[9]; + tensor.rank = p[11] == 0 ? p[10] : 0xFF; + for (size_t i = 0; i < 4; ++i) + tensor.shape[i] = read_i32(p + 12 + 4 * i); + const uint32_t scale_bits = read_u32(p + 28); + std::memcpy(&tensor.quant_scale, &scale_bits, sizeof(scale_bits)); + tensor.quant_zero_point = read_i32(p + 32); + return true; +} + +bool read_operation( + const GraphView& graph, + uint32_t index, + OperationRecord& operation) { + if (index >= graph.header.num_operations) + return false; + const uint8_t* p = graph.data + graph.operation_table_offset + + static_cast(index) * OPERATION_RECORD_SIZE; + operation.op_id = p[0]; + operation.opcode = p[1]; + operation.num_inputs = p[2]; + operation.num_outputs = p[3]; + for (size_t i = 0; i < 6; ++i) + operation.inputs[i] = read_u32(p + 4 + 4 * i); + for (size_t i = 0; i < 2; ++i) + operation.outputs[i] = read_u32(p + 28 + 4 * i); + operation.parameter_offset = read_u32(p + 36); + operation.parameter_size = read_u32(p + 40); + return true; +} + +const uint8_t* tensor_constant_data( + const GraphView& graph, + const TensorRecord& tensor) { + return graph.constant_data + tensor.constant_offset; +} + +bool parse_graph(const uint8_t* data, size_t size, GraphView& graph) { + graph = {}; + if (data == nullptr || size < GRAPH_HEADER_SIZE) { + std::printf("ERROR: graph is null or shorter than the 22-byte header\n"); + return false; + } + if (std::memcmp(data, "XTS3", 4) != 0) { + std::printf("ERROR: invalid graph magic\n"); + return false; + } + graph.header.magic[0] = 'X'; + graph.header.magic[1] = 'T'; + graph.header.magic[2] = 'S'; + graph.header.magic[3] = '3'; + graph.header.version = read_u16(data + 4); + graph.header.num_tensors = read_u32(data + 6); + graph.header.num_operations = read_u32(data + 10); + graph.header.parameter_size = read_u32(data + 14); + graph.header.constant_size = read_u32(data + 18); + if (!is_int8_graph_version(graph.header.version) || + graph.header.num_tensors == 0 || + graph.header.num_tensors > MAX_GRAPH_TENSORS || + graph.header.num_operations == 0 || + graph.header.num_operations > MAX_GRAPH_OPERATIONS) { + std::printf("ERROR: unsupported version or graph record counts\n"); + return false; + } + + graph.data = data; + graph.size = size; + graph.tensor_table_offset = GRAPH_HEADER_SIZE; + if (graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION) { + if (!range_valid(GRAPH_HEADER_SIZE, GRAPH_IO_TABLE_HEADER_SIZE, size)) + return false; + graph.num_inputs = read_u16(data + GRAPH_HEADER_SIZE); + graph.num_outputs = read_u16(data + GRAPH_HEADER_SIZE + 2); + const size_t io_bytes = + 4U * (static_cast(graph.num_inputs) + graph.num_outputs); + if (graph.num_inputs == 0 || graph.num_outputs == 0 || + graph.num_inputs > MAX_GRAPH_IO || graph.num_outputs > MAX_GRAPH_IO || + !range_valid( + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE, io_bytes, size)) { + std::printf("ERROR: graph IO table is invalid\n"); + return false; + } + graph.io_ids = data + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE; + for (uint32_t i = 0; + i < static_cast(graph.num_inputs) + graph.num_outputs; + ++i) { + if (read_u32(graph.io_ids + 4 * i) >= graph.header.num_tensors) + return false; + } + graph.tensor_table_offset = + GRAPH_HEADER_SIZE + GRAPH_IO_TABLE_HEADER_SIZE + io_bytes; + } + graph.tensor_record_size = TENSOR_RECORD_SIZE; + const size_t tensor_bytes = + static_cast(graph.header.num_tensors) * graph.tensor_record_size; + if (!range_valid(graph.tensor_table_offset, tensor_bytes, size)) + return false; + graph.operation_table_offset = graph.tensor_table_offset + tensor_bytes; + const size_t operation_bytes = + static_cast(graph.header.num_operations) * OPERATION_RECORD_SIZE; + if (!range_valid(graph.operation_table_offset, operation_bytes, size)) + return false; + graph.parameter_offset = graph.operation_table_offset + operation_bytes; + if (!range_valid(graph.parameter_offset, graph.header.parameter_size, size)) + return false; + graph.constant_offset = graph.parameter_offset + graph.header.parameter_size; + if (!range_valid(graph.constant_offset, graph.header.constant_size, size) || + graph.constant_offset + graph.header.constant_size != size) { + std::printf("ERROR: graph sections exceed blob or leave trailing bytes\n"); + return false; + } + if ((graph.constant_offset % INT8_PARAMETER_ALIGNMENT) != 0) { + std::printf( + "ERROR: int8 constant region is not 16-byte aligned within the blob\n"); + return false; + } + graph.parameter_data = data + graph.parameter_offset; + graph.constant_data = data + graph.constant_offset; + + const unsigned allowed_flags = 0x0FU; + for (uint32_t i = 0; i < graph.header.num_tensors; ++i) { + TensorRecord tensor{}; + read_tensor(graph, i, tensor); + if (tensor.tensor_id != i || !valid_dtype(tensor.dtype) || + tensor.rank > 4 || (tensor.flags & ~allowed_flags) != 0) { + std::printf( + "ERROR: tensor record %u has invalid ID, dtype, rank, or flags\n", + static_cast(i)); + return false; + } + for (uint8_t d = 0; d < tensor.rank; ++d) { + if (tensor.shape[d] <= 0) + return false; + } + for (uint8_t d = tensor.rank; d < 4; ++d) { + if (tensor.shape[d] != 0) + return false; + } + const bool is_constant = (tensor.flags & TENSOR_FLAG_CONSTANT) != 0; + const bool has_quantization = (tensor.flags & TENSOR_FLAG_QUANTIZED) != 0; + const uint8_t quantized_dtype = static_cast(DataType::INT8); + if ((has_quantization && + (tensor.dtype != quantized_dtype || + !std::isfinite(tensor.quant_scale) || tensor.quant_scale <= 0.0f || + tensor.quant_zero_point < -128 || tensor.quant_zero_point > 127)) || + (!has_quantization && + (tensor.quant_scale != 0.0f || tensor.quant_zero_point != 0))) { + std::printf( + "ERROR: tensor %u quantization metadata is invalid\n", + static_cast(i)); + return false; + } + const bool quantized_constant_allowed = + graph.header.version == GRAPH_ESP_NN_INT8_IO_VERSION && + tensor.dtype == static_cast(DataType::INT8); + if ((is_constant + ? ((tensor.dtype != static_cast(DataType::INT8) && + tensor.dtype != static_cast(DataType::INT32)) || + (has_quantization && !quantized_constant_allowed) || + (tensor.flags & + (TENSOR_FLAG_GRAPH_INPUT | TENSOR_FLAG_GRAPH_OUTPUT)) != 0) + : (tensor.dtype != static_cast(DataType::INT8) || + !has_quantization))) { + std::printf( + "ERROR: tensor %u is not a supported native int8 tensor\n", + static_cast(i)); + return false; + } + const size_t data_size = tensor_data_size(tensor); + const bool misaligned_int32 = is_constant && + tensor.dtype == static_cast(DataType::INT32) && + (tensor.constant_offset % sizeof(int32_t)) != 0; + if (data_size == 0 || + tensor_element_count(tensor) > static_cast(INT32_MAX) || + misaligned_int32 || + (is_constant && + !range_valid( + tensor.constant_offset, data_size, graph.header.constant_size)) || + (!is_constant && tensor.constant_offset != 0)) { + std::printf( + "ERROR: tensor %u constant data range or shape is invalid\n", + static_cast(i)); + return false; + } + } + + size_t expected_parameter_offset = 0; + for (uint32_t i = 0; i < graph.header.num_operations; ++i) { + OperationRecord op{}; + read_operation(graph, i, op); + if (op.op_id != (i & 0xFFU) || + !valid_opcode(op.opcode, graph.header.version) || op.num_inputs == 0 || + op.num_inputs > 6 || op.num_outputs == 0 || op.num_outputs > 2 || + op.parameter_offset != expected_parameter_offset || + !range_valid( + op.parameter_offset, + op.parameter_size, + graph.header.parameter_size)) { + std::printf( + "ERROR: operation record %u is invalid\n", static_cast(i)); + return false; + } + for (uint8_t j = 0; j < op.num_inputs; ++j) + if (!valid_tensor_reference(graph, op.inputs[j])) + return false; + for (uint8_t j = 0; j < op.num_outputs; ++j) + if (!valid_tensor_reference(graph, op.outputs[j])) + return false; + for (uint8_t j = op.num_inputs; j < 6; ++j) + if (op.inputs[j] != 0) + return false; + for (uint8_t j = op.num_outputs; j < 2; ++j) + if (op.outputs[j] != 0) + return false; + if (!validate_int8_operation(graph, op)) { + std::printf( + "ERROR: operation %u (%s) metadata or parameter block is invalid\n", + static_cast(i), + opcode_name(static_cast(op.opcode))); + return false; + } + expected_parameter_offset += op.parameter_size; + } + const size_t parameter_padding = + graph.header.parameter_size - expected_parameter_offset; + if (expected_parameter_offset > graph.header.parameter_size || + parameter_padding >= INT8_PARAMETER_ALIGNMENT) { + std::printf( + "ERROR: parameter region size does not match operation records\n"); + return false; + } + if (!validate_int8_dataflow(graph)) + return false; + return true; +} + +const char* opcode_name(OpCode opcode) { + switch (opcode) { + case OpCode::CONV2D: + return "CONV2D"; + case OpCode::DEPTHWISE_CONV2D: + return "DEPTHWISE_CONV2D"; + case OpCode::RELU6: + return "RELU6"; + case OpCode::ADD: + return "ADD"; + case OpCode::AVGPOOL2D: + return "AVGPOOL2D"; + case OpCode::LINEAR: + return "LINEAR"; + case OpCode::FLATTEN: + return "FLATTEN"; + case OpCode::MAX_POOL2D: + return "MAX_POOL2D"; + case OpCode::AVG_POOL2D: + return "AVG_POOL2D"; + case OpCode::CONCAT: + return "CONCAT"; + case OpCode::LUT: + return "LUT"; + case OpCode::MUL: + return "MUL"; + case OpCode::SOFTMAX: + return "SOFTMAX"; + case OpCode::COPY: + return "COPY"; + case OpCode::UPSAMPLE: + return "UPSAMPLE"; + case OpCode::TRANSPOSE_CONV2D: + return "TRANSPOSE_CONV2D"; + default: + return "UNKNOWN"; + } +} + +} // namespace espressif +} // namespace backends +} // namespace executorch diff --git a/backends/espressif/lowering.py b/backends/espressif/lowering.py new file mode 100644 index 00000000000..67e4063b54d --- /dev/null +++ b/backends/espressif/lowering.py @@ -0,0 +1,36 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import executorch.kernels.quantized # noqa: F401 registers quantize/dequantize out variants + +from executorch.backends.espressif.partition.espnn_partitioner import EspNNPartitioner +from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower +from executorch.exir.backend.canonical_partitioners.duplicate_dequant_node_pass import ( + DuplicateDequantNodePass, +) + + +def to_espnn_edge(program, allow_cpu_fallback: bool = True): + """ + Lower a PT2E-quantized torch.export program for the ESP32 ExecuTorch + runtime: int8 regions become ESP-NN delegates, anything else stays in + the program for the portable CPU kernels. Returns (edge_manager, report). + """ + partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback) + edge = to_edge_transform_and_lower( + program, + transform_passes=[DuplicateDequantNodePass()], + partitioner=[partitioner], + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + return edge, partitioner.report + + +__all__ = ["to_espnn_edge"] diff --git a/backends/espressif/partition/__init__.py b/backends/espressif/partition/__init__.py new file mode 100644 index 00000000000..3b10d5189d8 --- /dev/null +++ b/backends/espressif/partition/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .espnn_partitioner import EspNNPartitioner + +__all__ = ["EspNNPartitioner"] diff --git a/backends/espressif/partition/espnn_partitioner.py b/backends/espressif/partition/espnn_partitioner.py new file mode 100644 index 00000000000..c1aa0de730e --- /dev/null +++ b/backends/espressif/partition/espnn_partitioner.py @@ -0,0 +1,400 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import Callable, Optional + +import torch + +from executorch.backends.espressif.espnn_preprocess import ( + EspNNBackend, + INT8_FORMAT_SPEC, +) +from executorch.backends.espressif.serialization.int8_ops import ( + consumers, + is_dequantize, + is_movement_terminal, + is_quantize, + MOVEMENT_KINDS, + node_kind, + SPLIT_KINDS, +) +from executorch.backends.espressif.serialization.xts3 import ( + _target_kind, + ExportedTensorResolver, + INT8_PASSTHROUGH_MARKERS, + lower_int8_node, + LUT_KINDS, + TensorRegistry, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec +from executorch.exir.backend.partitioner import ( + DelegationSpec, + Partitioner, + PartitionResult, +) +from executorch.exir.backend.utils import tag_constant_data +from torch.export import ExportedProgram +from torch.fx.passes.infra.partitioner import CapabilityBasedPartitioner +from torch.fx.passes.operator_support import OperatorSupportBase +from torch.fx.passes.utils.fuser_utils import validate_partition + + +SUPPORTED_KINDS = { + "conv2d", + "convolution", + "conv_transpose2d", + "hardtanh", + "hardtanh_", + "relu", + "relu6", + "add", + "adaptive_avg_pool2d", + "mean_dim", + "max_pool2d", + "max_pool2d_with_indices", + "avg_pool2d", + "cat", + "stack", + "flatten", + "reshape", + "linear", + "identity", + "dropout", + "getitem", + "mul", + "softmax", + "pad", + "upsample_nearest2d", + "upsample_nearest2d_vec", + "upsample_bilinear2d", + "upsample_bilinear2d_vec", + *MOVEMENT_KINDS, + *SPLIT_KINDS, + *LUT_KINDS, +} + +PRESERVED_OPS = [ + torch.ops.aten.linear.default, + torch.ops.aten.hardtanh.default, + torch.ops.aten.max_pool2d.default, + torch.ops.aten.flatten.using_ints, + torch.ops.aten.adaptive_avg_pool2d.default, + torch.ops.aten.avg_pool2d.default, + torch.ops.aten.hardswish.default, + torch.ops.aten.hardsigmoid.default, + torch.ops.aten.silu.default, + torch.ops.aten.upsample_nearest2d.vec, + torch.ops.aten.upsample_bilinear2d.vec, +] + + +def is_supported_int8_node(node) -> bool: + if node.op != "call_function": + return False + text = str(node.target) + if any(marker in text for marker in INT8_PASSTHROUGH_MARKERS): + return True + return _target_kind(node.target) in SUPPORTED_KINDS + + +def _keep_if_quantized(node: torch.fx.Node) -> bool: + """Only quantized instances stay whole; float ones decompose for the CPU.""" + tensor_inputs = [arg for arg in node.args if isinstance(arg, torch.fx.Node)] + if not tensor_inputs or not is_dequantize(tensor_inputs[0]): + return False + if not any(is_quantize(user) for user in node.users) and not any( + _target_kind(user.target) in ("relu", "relu6", "hardtanh", "hardtanh_") + and any(is_quantize(u) for u in user.users) + for user in node.users + if user.op == "call_function" + ): + return False + if _target_kind(node.target) == "linear": + value = tensor_inputs[0].meta.get("val") + if value is None or value.dim() == 4 or value.dim() < 1: + return False + return True + + +def _operator_name(target) -> str: + import re + + match = re.search( + r"([A-Za-z_]+(?:\.[A-Za-z0-9_]+){1,2})(?=[>:\s]|$)", + str(target).replace(" bool: + return node.op == "call_function" and any( + marker in str(node.target) for marker in INT8_PASSTHROUGH_MARKERS + ) + + +class StructuralResolver(ExportedTensorResolver): + """ + ExecuTorch partitions a copy of the program whose constants are + FakeTensors. Support is judged on shapes, arguments, and quantization + parameters, so fake constants are replaced by neutral stand-ins of the + same shape and dtype (scales of one, zero points and weights of zero). + """ + + def resolve(self, node): + from torch._subclasses.fake_tensor import FakeTensor + + value = super().resolve(node) + if not isinstance(value, FakeTensor): + return value + shape = tuple(int(d) for d in value.shape) + if value.dtype.is_floating_point: + return torch.ones(shape, dtype=value.dtype) + return torch.zeros(shape, dtype=value.dtype) + + +def find_supported_nodes(exported_program: ExportedProgram) -> tuple[set, dict]: + graph_module = exported_program.graph_module + registry = TensorRegistry(StructuralResolver(exported_program), graph_module) + supported = set() + reasons = {} + for node in graph_module.graph.nodes: + if node.op != "call_function": + continue + if _is_passthrough(node): + supported.add(node) + continue + if _target_kind(node.target) not in SUPPORTED_KINDS: + reasons[node] = "no int8 lowering for this operator" + continue + fused_before = set(registry.fused_nodes) + try: + registry.extra_operations.clear() + lowered = lower_int8_node(node, registry) + if lowered is not None or node in registry.fused_nodes: + supported.add(node) + else: + reasons[node] = "operator was not lowered" + except Exception as error: + registry.fused_nodes = fused_before + reasons[node] = str(error).splitlines()[0][:160] + registry.extra_operations.clear() + + changed = True + while changed: + changed = False + for node in list(supported): + kind = node_kind(node) + deferred = ( + kind in MOVEMENT_KINDS and not is_movement_terminal(node) + ) or kind in SPLIT_KINDS + if not deferred: + continue + downstream = ( + consumers(node) if kind not in SPLIT_KINDS else list(node.users) + ) + if any(user not in supported for user in downstream): + supported.discard(node) + reasons[node] = "data movement feeds an operator that runs on the CPU" + changed = True + return supported, reasons + + +class _SupportSet(OperatorSupportBase): + def __init__(self, nodes: set): + self.nodes = nodes + + def is_node_supported(self, submodules, node: torch.fx.Node) -> bool: + return node in self.nodes + + +def _is_constant(node, resolver: ExportedTensorResolver) -> bool: + return node.op in ("placeholder", "get_attr") and resolver.resolve(node) is not None + + +def _boundary_is_valid(nodes: set, resolver: ExportedTensorResolver) -> bool: + for node in nodes: + for user in node.users: + if user not in nodes and not (is_quantize(node) or is_dequantize(node)): + return False + for source in node.all_input_nodes: + if source in nodes or _is_constant(source, resolver): + continue + if not (is_quantize(node) or is_dequantize(node)): + return False + return True + + +FUSABLE_ACTIVATION_KINDS = ("relu", "relu6", "hardtanh", "hardtanh_") + + +def _attach_outputs_to_producers(partitions: list[set]) -> None: + """ + A quantize node (and an activation fused into its producer) must live in + the producer's partition so the partition exports int8 or dequantized + values only, never a raw accumulator. A move that would make either + partition non-convex is skipped; the affected partition then fails + _boundary_is_valid and stays on the CPU instead of breaking fusion. + """ + owner = {} + for index, nodes in enumerate(partitions): + for node in nodes: + owner[node] = index + changed = True + while changed: + changed = False + for index, nodes in enumerate(partitions): + for node in list(nodes): + if _is_passthrough(node): + continue + for user in node.users: + attach = is_quantize(user) or ( + user.op == "call_function" + and _target_kind(user.target) in FUSABLE_ACTIVATION_KINDS + and user in owner + ) + if not attach or owner.get(user) == index or user not in owner: + continue + donor = partitions[owner[user]] + if not validate_partition( + list(nodes | {user}) + ) or not validate_partition(list(donor - {user})): + continue + donor.discard(user) + nodes.add(user) + owner[user] = index + changed = True + + +def _has_compute(nodes: set) -> bool: + return any(not _is_passthrough(node) for node in nodes) + + +class EspNNPartitioner(Partitioner): + """ + Delegates PT2E-quantized regions of a graph to the ESP32 ESP-NN backend. + + With allow_cpu_fallback (the default) operators the int8 backend cannot + lower stay in the ExecuTorch program and run on the CPU through the + portable kernels; partition boundaries carry float or int8 tensors. + With allow_cpu_fallback=False any operator left outside a delegate is an + error, whether it is unsupported or its region could not be delegated; + the standalone XTS3 firmware path requires this. + """ + + def __init__( + self, + compile_specs: Optional[list[CompileSpec]] = None, + allow_cpu_fallback: bool = True, + ): + specs = list(compile_specs or []) + if not any(spec.key == INT8_FORMAT_SPEC.key for spec in specs): + specs.append(INT8_FORMAT_SPEC) + self.delegation_spec = DelegationSpec(EspNNBackend.__name__, specs) + self.allow_cpu_fallback = allow_cpu_fallback + self.report: dict = {} + + def ops_to_not_decompose( + self, ep: ExportedProgram + ) -> tuple[list[torch._ops.OpOverload], Optional[Callable[[torch.fx.Node], bool]]]: + return PRESERVED_OPS, _keep_if_quantized + + def partition(self, exported_program: ExportedProgram) -> PartitionResult: + from torch._subclasses.fake_tensor import unset_fake_temporarily + + graph_module = exported_program.graph_module + with unset_fake_temporarily(): + supported, reasons = find_supported_nodes(exported_program) + unsupported = [ + node + for node in graph_module.graph.nodes + if node.op == "call_function" and node not in supported + ] + if unsupported and not self.allow_cpu_fallback: + raise RuntimeError( + "The ESP32 int8 backend cannot run these operators: " + + ", ".join( + f"{node.name} ({node.target}): {reasons.get(node, '')}" + for node in unsupported + ) + ) + + with unset_fake_temporarily(): + resolver = StructuralResolver(exported_program) + proposer = CapabilityBasedPartitioner( + graph_module, _SupportSet(supported), allows_single_node_partition=True + ) + partitions = [ + set(partition.nodes) for partition in proposer.propose_partitions() + ] + _attach_outputs_to_producers(partitions) + tags = {} + rejected = 0 + for nodes in partitions: + if not _has_compute(nodes): + continue + with unset_fake_temporarily(): + valid = _boundary_is_valid(nodes, resolver) + if not valid: + rejected += 1 + continue + tag = f"espnn_{len(tags)}" + for node in nodes: + node.meta["delegation_tag"] = tag + tags[tag] = self.delegation_spec + delegated = { + node + for node in graph_module.graph.nodes + if "delegation_tag" in node.meta and node.op == "call_function" + } + cpu_ops = [ + node + for node in graph_module.graph.nodes + if node.op == "call_function" + and node not in delegated + and not _is_passthrough(node) + ] + leftover = [node for node in cpu_ops if "getitem" not in str(node.target)] + if leftover and not self.allow_cpu_fallback: + raise RuntimeError( + f"{len(leftover)} operator(s) would run on the CPU " + f"({rejected} region(s) could not be delegated): " + + ", ".join( + f"{node.name} ({_operator_name(node.target)}): " + f"{reasons.get(node, 'outside the delegated regions')}" + for node in leftover[:20] + ) + ) + if not tags and not self.allow_cpu_fallback: + raise RuntimeError("No ESP32 int8 partition could be formed") + tag_constant_data(exported_program) + self.report = { + "partitions": len(tags), + "rejected_partitions": rejected, + "delegated_ops": sum(1 for node in delegated if not _is_passthrough(node)), + "cpu_ops": [ + { + "name": node.name, + "target": _operator_name(node.target), + "reason": reasons.get(node, "outside the delegated regions"), + } + for node in leftover + ], + } + return PartitionResult( + tagged_exported_program=exported_program, partition_tags=tags + ) + + +__all__ = [ + "StructuralResolver", + "EspNNPartitioner", + "find_supported_nodes", + "is_supported_int8_node", +] diff --git a/backends/espressif/quantizer/__init__.py b/backends/espressif/quantizer/__init__.py new file mode 100644 index 00000000000..60a6bccfd8d --- /dev/null +++ b/backends/espressif/quantizer/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .espnn_quantizer import EspNNQuantizer + +__all__ = ["EspNNQuantizer"] diff --git a/backends/espressif/quantizer/espnn_quantizer.py b/backends/espressif/quantizer/espnn_quantizer.py new file mode 100644 index 00000000000..1bebb13eb2b --- /dev/null +++ b/backends/espressif/quantizer/espnn_quantizer.py @@ -0,0 +1,428 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import Callable, Optional, Union + +import torch + +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( + _get_module_type_filter, + get_symmetric_quantization_config, + XNNPACKQuantizer, +) +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer_utils import ( + _is_share_obs_or_fq_op, +) +from torch.fx import Node +from torchao.quantization.pt2e.quantizer import ( + FixedQParamsQuantizationSpec, + get_bias_qspec, + get_input_act_qspec, + get_module_name_filter, + get_output_act_qspec, + get_weight_qspec, + QuantizationAnnotation, + QuantizationConfig, + SharedQuantizationSpec, +) +from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY + + +CONV_TARGETS = (torch.ops.aten.conv2d.default,) +LINEAR_TARGETS = (torch.ops.aten.linear.default,) +ADD_TARGETS = (torch.ops.aten.add.Tensor, torch.ops.aten.add_.Tensor) +RELU_TARGETS = (torch.ops.aten.relu.default, torch.ops.aten.relu_.default) +HARDTANH_TARGETS = (torch.ops.aten.hardtanh.default, torch.ops.aten.hardtanh_.default) +RELU6_TARGETS = (torch.ops.aten.relu6.default,) +LUT_TARGETS = ( + torch.ops.aten.hardswish.default, + torch.ops.aten.hardswish_.default, + torch.ops.aten.hardsigmoid.default, + torch.ops.aten.hardsigmoid_.default, + torch.ops.aten.sigmoid.default, + torch.ops.aten.sigmoid_.default, + torch.ops.aten.silu.default, + torch.ops.aten.silu_.default, + torch.ops.aten.tanh.default, + torch.ops.aten.tanh_.default, + torch.ops.aten.gelu.default, + torch.ops.aten.elu.default, + torch.ops.aten.elu_.default, + torch.ops.aten.leaky_relu.default, + torch.ops.aten.leaky_relu_.default, +) +SOFTMAX_TARGETS = (torch.ops.aten.softmax.int, torch.ops.aten._softmax.default) +SOFTMAX_OUTPUT_SCALE = 1.0 / 256.0 +SOFTMAX_OUTPUT_ZERO_POINT = -128 + + +def fused_activation_kind(node: Node) -> Optional[str]: + if node.op != "call_function": + return None + if node.target in RELU_TARGETS: + return "relu" + if node.target in RELU6_TARGETS: + return "relu6" + if node.target in HARDTANH_TARGETS: + minimum = float( + node.args[1] if len(node.args) > 1 else node.kwargs.get("min_val", -1.0) + ) + maximum = float( + node.args[2] if len(node.args) > 2 else node.kwargs.get("max_val", 1.0) + ) + if minimum == 0.0 and maximum == 6.0: + return "relu6" + return None + + +def _is_annotated(node: Node) -> bool: + annotation = node.meta.get(Q_ANNOTATION_KEY) + return annotation is not None and annotation._annotated + + +def _mark_annotated(nodes) -> None: + for node in nodes: + if node is None: + continue + if Q_ANNOTATION_KEY not in node.meta: + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation() + node.meta[Q_ANNOTATION_KEY]._annotated = True + + +def _annotate_weighted_activation( + producer: Node, + activation: Node, + config: QuantizationConfig, +) -> bool: + input_act = producer.args[0] + weight = producer.args[1] + bias = producer.args[2] if len(producer.args) > 2 else None + if not isinstance(input_act, Node) or not isinstance(weight, Node): + return False + nodes = [producer, activation, weight] + if isinstance(bias, Node): + nodes.append(bias) + if any(_is_annotated(node) for node in nodes): + return False + + input_qspec_map = { + input_act: get_input_act_qspec(config), + weight: get_weight_qspec(config), + } + if isinstance(bias, Node): + input_qspec_map[bias] = get_bias_qspec(config) + + producer.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map=input_qspec_map, + _annotated=True, + ) + activation.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + output_qspec=get_output_act_qspec(config), + _annotated=True, + ) + _mark_annotated(nodes) + return True + + +def _annotate_add_activation( + add: Node, + activation: Node, + config: QuantizationConfig, +) -> bool: + lhs = add.args[0] + rhs = add.args[1] + if not isinstance(lhs, Node) or not isinstance(rhs, Node): + return False + if _is_annotated(add) or _is_annotated(activation): + return False + add.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={ + lhs: get_input_act_qspec(config), + rhs: get_input_act_qspec(config), + }, + _annotated=True, + ) + activation.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + output_qspec=get_output_act_qspec(config), + _annotated=True, + ) + return True + + +ConfigSource = Union[ + QuantizationConfig, Callable[[Node], Optional[QuantizationConfig]], None +] + + +def _config_lookup( + config: ConfigSource, +) -> Callable[[Node], Optional[QuantizationConfig]]: + if callable(config) and not isinstance(config, QuantizationConfig): + return config + return lambda node: config + + +def annotate_fused_activations( + model: torch.fx.GraphModule, + config: ConfigSource, +) -> int: + """ + Annotate conv/linear/add followed by ReLU or ReLU6 as one fused unit. + `config` is a QuantizationConfig or a per-node lookup returning one, or + None for nodes that must stay in float. + """ + config_for = _config_lookup(config) + fused = 0 + for node in model.graph.nodes: + if fused_activation_kind(node) is None: + continue + producer = node.args[0] + if not isinstance(producer, Node) or producer.op != "call_function": + continue + if len(producer.users) != 1: + continue + config = config_for(node) + if config is None or config_for(producer) is not config: + continue + if producer.target in CONV_TARGETS or producer.target in LINEAR_TARGETS: + fused += int(_annotate_weighted_activation(producer, node, config)) + elif producer.target in ADD_TARGETS: + fused += int(_annotate_add_activation(producer, node, config)) + return fused + + +class EspNNQuantizer(XNNPACKQuantizer): + """ + PT2E quantizer matching the ESP-NN int8 kernel contract. + + Activations: int8 per-tensor affine. + Weights: int8 per-output-channel symmetric. + ReLU and ReLU6 that directly follow conv2d, linear, or add are fused + into the producer so that the clamp happens on the producer's + requantized output, exactly as the ESP-NN activation_min/max arguments + implement it. Without this fusion ReLU6 would inherit the producer's + full-range scale and waste most of the int8 grid. + """ + + def __init__(self, per_channel: bool = True, is_qat: bool = False): + super().__init__() + self.set_global( + get_symmetric_quantization_config(is_per_channel=per_channel, is_qat=is_qat) + ) + self.fused_activation_count = 0 + self.rewritten_split_count = 0 + + def node_config(self, node: Node) -> Optional[QuantizationConfig]: + """ + The config that governs `node`, with the same precedence as + XNNPACKQuantizer: module name, then module type, then global. + None means the node stays in float. + """ + for name, config in self.module_name_config.items(): + if get_module_name_filter(name)(node): + return config + for module_type, config in self.module_type_config.items(): + if _get_module_type_filter(module_type)(node): + return config + return self.global_config + + def transform_for_annotation( + self, model: torch.fx.GraphModule + ) -> torch.fx.GraphModule: + model = super().transform_for_annotation(model) + self.rewritten_split_count = rewrite_splits_as_slices(model) + return model + + def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule: + self.fused_activation_count = annotate_fused_activations( + model, self.node_config + ) + model = self._annotate_for_quantization_config(model) + annotate_unary_and_softmax(model, self.node_config) + propagate_shared_quantization(model, self.node_config) + return model + + +def annotate_unary_and_softmax( + model: torch.fx.GraphModule, config: ConfigSource +) -> None: + config_for = _config_lookup(config) + softmax_output = FixedQParamsQuantizationSpec( + dtype=torch.int8, + scale=SOFTMAX_OUTPUT_SCALE, + zero_point=SOFTMAX_OUTPUT_ZERO_POINT, + quant_min=-128, + quant_max=127, + qscheme=torch.per_tensor_affine, + ) + for node in model.graph.nodes: + if node.op != "call_function" or _is_annotated(node): + continue + if node.target not in LUT_TARGETS and node.target not in SOFTMAX_TARGETS: + continue + source = node.args[0] + config = config_for(node) + if not isinstance(source, Node) or config is None: + continue + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={source: get_input_act_qspec(config)}, + output_qspec=( + softmax_output + if node.target in SOFTMAX_TARGETS + else get_output_act_qspec(config) + ), + _annotated=True, + ) + + +EXTRA_SHARED_QPARAM_TARGETS = ( + torch.ops.aten.avg_pool2d.default, + torch.ops.aten.relu6.default, + torch.ops.aten.unsqueeze.default, + torch.ops.aten.squeeze.default, + torch.ops.aten.squeeze.dims, + torch.ops.aten.slice.Tensor, + torch.ops.aten.select.int, + torch.ops.aten.expand.default, + torch.ops.aten.unflatten.int, + torch.ops.aten._unsafe_view.default, + torch.ops.aten.clone.default, + torch.ops.aten.contiguous.default, + torch.ops.aten.upsample_nearest2d.vec, + torch.ops.aten.upsample_nearest2d.default, + torch.ops.aten.upsample_bilinear2d.vec, + torch.ops.aten.upsample_bilinear2d.default, + torch.ops.aten.constant_pad_nd.default, + torch.ops.aten.pad.default, + torch.ops.aten.dropout.default, + torch.ops.aten.dropout_.default, + torch.ops.aten.alias.default, +) + +SPLIT_TARGETS = ( + torch.ops.aten.split.Tensor, + torch.ops.aten.split_with_sizes.default, + torch.ops.aten.chunk.default, + torch.ops.aten.unbind.int, +) + + +def rewrite_splits_as_slices(model: torch.fx.GraphModule) -> int: + """ + Replace split/chunk/unbind + getitem with slice/select so that every + piece is an ordinary tensor that can carry a shared quantization spec. + """ + import operator + + rewritten = 0 + for node in list(model.graph.nodes): + if node.op != "call_function" or node.target not in SPLIT_TARGETS: + continue + source = node.args[0] + value = source.meta.get("val") if isinstance(source, Node) else None + if value is None: + continue + users = list(node.users) + if any(user.target is not operator.getitem for user in users): + continue + rank = value.dim() + if node.target is torch.ops.aten.unbind.int: + dim = int(node.args[1]) if len(node.args) > 1 else 0 + dim %= rank + bounds = None + else: + dim = ( + int(node.args[2]) + if len(node.args) > 2 + else int(node.kwargs.get("dim", 0)) + ) + dim %= rank + size = int(value.shape[dim]) + if node.target is torch.ops.aten.chunk.default: + chunks = int(node.args[1]) + step = (size + chunks - 1) // chunks + sizes = [min(step, size - start) for start in range(0, size, step)] + elif isinstance(node.args[1], (list, tuple)): + sizes = [int(v) for v in node.args[1]] + else: + step = int(node.args[1]) + sizes = [min(step, size - start) for start in range(0, size, step)] + starts = [sum(sizes[:i]) for i in range(len(sizes))] + bounds = [(start, start + length) for start, length in zip(starts, sizes)] + for user in users: + index = int(user.args[1]) + with model.graph.inserting_before(user): + if bounds is None: + replacement = model.graph.call_function( + torch.ops.aten.select.int, (source, dim, index) + ) + else: + start, end = bounds[index] + replacement = model.graph.call_function( + torch.ops.aten.slice.Tensor, (source, dim, start, end, 1) + ) + replacement.meta = dict(user.meta) + user.replace_all_uses_with(replacement) + model.graph.erase_node(user) + model.graph.erase_node(node) + rewritten += 1 + if rewritten: + model.graph.lint() + model.recompile() + return rewritten + + +_ALWAYS = object() + + +def _shares_qparams(node: Node) -> bool: + return node.op == "call_function" and ( + _is_share_obs_or_fq_op(node.target) + or node.target in EXTRA_SHARED_QPARAM_TARGETS + ) + + +def propagate_shared_quantization( + model: torch.fx.GraphModule, config: ConfigSource = _ALWAYS +) -> None: + """ + Give shape and movement ops their input's quantization. With a per-node + `config` lookup, nodes whose config is None are left in float. + """ + config_for = _config_lookup(config) + for node in model.graph.nodes: + if not _shares_qparams(node): + continue + if _is_annotated(node) or config_for(node) is None: + continue + source = node.args[0] + if not isinstance(source, Node): + continue + annotation = source.meta.get(Q_ANNOTATION_KEY) + if annotation is None or annotation.output_qspec is None: + continue + shared = SharedQuantizationSpec(source) + node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation( + input_qspec_map={source: shared}, + output_qspec=shared, + _annotated=True, + ) + + +__all__ = [ + "EspNNQuantizer", + "annotate_fused_activations", + "fused_activation_kind", + "propagate_shared_quantization", + "annotate_unary_and_softmax", + "rewrite_splits_as_slices", + "SOFTMAX_OUTPUT_SCALE", + "SOFTMAX_OUTPUT_ZERO_POINT", +] diff --git a/backends/espressif/reference/__init__.py b/backends/espressif/reference/__init__.py new file mode 100644 index 00000000000..79d9f63d14a --- /dev/null +++ b/backends/espressif/reference/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .esp_nn_reference import ReferenceInterpreter +from .program_emulator import ProgramEmulator + +__all__ = ["ProgramEmulator", "ReferenceInterpreter"] diff --git a/backends/espressif/reference/esp_nn_reference.py b/backends/espressif/reference/esp_nn_reference.py new file mode 100644 index 00000000000..291ccca80f7 --- /dev/null +++ b/backends/espressif/reference/esp_nn_reference.py @@ -0,0 +1,643 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +import numpy as np + +from ..serialization.xts3 import ( + COPY_FLAG_FILL, + COPY_HEADER, + COPY_HEADER_SIZE, + COPY_MAX_RANK, + COPY_SEGMENT, + COPY_SEGMENT_SIZE, + INT8_ADD_PARAMS, + INT8_CONV_BASE_PARAMS, + INT8_CONV_BASE_PARAMS_SIZE, + INT8_MUL_PARAMS, + INT8_PAIR_PARAMS, + INT8_PAIR_PARAMS_SIZE, + INT8_POOL_PARAMS, + MUL_MODE_CHANNEL_BROADCAST, + SOFTMAX_OUTPUT_ZERO_POINT, + TENSOR_FLAG_GRAPH_INPUT, + TENSOR_FLAG_GRAPH_OUTPUT, + TRANSPOSE_CONV_PARAMS, + TRANSPOSE_CONV_PARAMS_SIZE, + UPSAMPLE_HEADER, + UPSAMPLE_HEADER_SIZE, + UPSAMPLE_WEIGHT_BITS, + XTS3_VERSIONS, + XTS3DType, + XTS3Graph, + XTS3OpCode, + XTS3Tensor, +) + + +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 + + +def wrap_int32(values: np.ndarray) -> np.ndarray: + values = np.asarray(values, dtype=np.int64) + return ((values + (1 << 31)) & 0xFFFFFFFF) - (1 << 31) + + +def saturating_rounding_doubling_high_mul(a, b) -> np.ndarray: + a = np.asarray(a, dtype=np.int64) + b = np.asarray(b, dtype=np.int64) + overflow = (a == b) & (a == INT32_MIN) + nudge = np.where((a < 0) ^ (b < 0), 1 - (1 << 30), 1 << 30) + product = a * b + nudge + high = np.where(product >= 0, product >> 31, -((-product) >> 31)) + return np.where(overflow, INT32_MAX, high) + + +def rounding_divide_by_power_of_two(x, exponent) -> np.ndarray: + x = np.asarray(x, dtype=np.int64) + exponent = np.asarray(exponent, dtype=np.int64) + mask = (np.int64(1) << exponent) - 1 + remainder = x & mask + result = x >> exponent + threshold = (mask >> 1) + (result < 0) + return result + (remainder > threshold) + + +def multiply_by_quantized_multiplier(x, multiplier, shift) -> np.ndarray: + shift = np.asarray(shift, dtype=np.int64) + multiplier = np.asarray(multiplier, dtype=np.int64) + left = np.maximum(shift, 0) + right = np.maximum(-shift, 0) + shifted = wrap_int32(np.asarray(x, dtype=np.int64) << left) + return rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(shifted, multiplier), right + ) + + +def _exact_matmul(a: np.ndarray, b: np.ndarray) -> np.ndarray: + result = a.astype(np.float64) @ b.astype(np.float64) + if result.size and np.max(np.abs(result)) >= float(1 << 52): + raise OverflowError("Accumulator exceeds the exactly representable range") + return np.rint(result).astype(np.int64) + + +def device_shape(tensor: XTS3Tensor) -> tuple[int, ...]: + shape = tuple(tensor.shape) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError( + f"Tensor {tensor.tensor_id} has batch {shape[0]}; only batch 1 is supported" + ) + return (shape[2], shape[3], shape[1]) + if len(shape) == 2 and shape[0] == 1: + return (shape[1],) + return shape + + +def to_device_layout(tensor: XTS3Tensor, logical: np.ndarray) -> np.ndarray: + logical = np.asarray(logical) + if logical.shape != tuple(tensor.shape): + raise ValueError(f"Expected shape {tensor.shape}, got {logical.shape}") + if len(tensor.shape) == 4: + return np.ascontiguousarray(logical[0].transpose(1, 2, 0)) + return np.ascontiguousarray(logical.reshape(device_shape(tensor))) + + +def to_logical_layout(tensor: XTS3Tensor, device: np.ndarray) -> np.ndarray: + if len(tensor.shape) == 4: + return np.ascontiguousarray(device.transpose(2, 0, 1)[None]) + return np.ascontiguousarray(device.reshape(tensor.shape)) + + +def quantize(tensor: XTS3Tensor, values: np.ndarray) -> np.ndarray: + inverse = np.float32(1.0 / tensor.quant_scale) + scaled = np.asarray(values, dtype=np.float32) * inverse + quantized = np.rint(scaled).astype(np.int64) + tensor.quant_zero_point + return np.clip(quantized, -128, 127).astype(np.int8) + + +def dequantize(tensor: XTS3Tensor, values: np.ndarray) -> np.ndarray: + return ( + values.astype(np.float32) - np.float32(tensor.quant_zero_point) + ) * np.float32(tensor.quant_scale) + + +@dataclass +class ConvParams: + kernel_h: int + kernel_w: int + stride_h: int + stride_w: int + pad_h: int + pad_w: int + groups: int + act_min: int + act_max: int + multipliers: np.ndarray + shifts: np.ndarray + dilation_h: int = 1 + dilation_w: int = 1 + + +def requantize(values, multiplier, shift, in_zero_point, out_zero_point) -> np.ndarray: + shifted = np.asarray(values, dtype=np.int64) - in_zero_point + result = ( + multiply_by_quantized_multiplier(shifted, multiplier, shift) + out_zero_point + ) + return np.clip(result, -128, 127) + + +def run_copy( + params: bytes, sources: list[np.ndarray], output: np.ndarray +) -> np.ndarray: + count, flags, fill = struct.unpack_from(COPY_HEADER, params, 0) + flat = output.reshape(-1).copy() + if flags & COPY_FLAG_FILL: + flat[:] = fill + offset = COPY_HEADER_SIZE + for _ in range(count): + fields = struct.unpack_from(COPY_SEGMENT, params, offset) + offset += COPY_SEGMENT_SIZE + index, rank, in_offset, out_offset = fields[:4] + dims = fields[4 : 4 + COPY_MAX_RANK][:rank] + in_strides = fields[4 + COPY_MAX_RANK : 4 + 2 * COPY_MAX_RANK][:rank] + out_strides = fields[4 + 2 * COPY_MAX_RANK : 4 + 3 * COPY_MAX_RANK][:rank] + requant, multiplier, shift, in_zp, out_zp = fields[4 + 3 * COPY_MAX_RANK :] + grids = np.meshgrid( + *[np.arange(d, dtype=np.int64) for d in dims], indexing="ij" + ) + in_index = np.full(grids[0].shape, in_offset, dtype=np.int64) + out_index = np.full(grids[0].shape, out_offset, dtype=np.int64) + for grid, in_stride, out_stride in zip(grids, in_strides, out_strides): + in_index += grid * in_stride + out_index += grid * out_stride + values = sources[index].reshape(-1)[in_index.reshape(-1)].astype(np.int64) + if requant: + values = requantize(values, multiplier, shift, in_zp, out_zp) + flat[out_index.reshape(-1)] = values + return flat.reshape(output.shape) + + +def parse_conv_params(params: bytes, channels: int) -> ConvParams: + expected = INT8_CONV_BASE_PARAMS_SIZE + 8 * channels + if len(params) != expected: + raise ValueError( + f"Convolution parameters are {len(params)} bytes, expected {expected}" + ) + kh, kw, sh, sw, ph, pw, dh, dw, groups, act_min, act_max = struct.unpack_from( + INT8_CONV_BASE_PARAMS, params, 0 + ) + base = INT8_CONV_BASE_PARAMS_SIZE + multipliers = np.frombuffer( + params, dtype=" np.ndarray: + tensor = self.graph.tensors[tensor_id] + dtype = {XTS3DType.INT8: np.int8, XTS3DType.INT32: " np.ndarray: + if tensor_id in values: + return values[tensor_id] + tensor = self.graph.tensors[tensor_id] + if tensor.is_constant: + return ( + self.constant(tensor_id).astype(np.int8).reshape(device_shape(tensor)) + ) + raise KeyError(f"Tensor {tensor_id} has not been produced") + + def run(self, inputs: dict[int, np.ndarray]) -> dict[int, np.ndarray]: + values: dict[int, np.ndarray] = {} + for tensor_id in self.inputs: + tensor = self.graph.tensors[tensor_id] + data = np.asarray(inputs[tensor_id], dtype=np.int8) + if data.shape != device_shape(tensor): + raise ValueError( + f"Input {tensor_id} must have device shape {device_shape(tensor)}" + ) + values[tensor_id] = data + for operation in self.graph.operations: + values[operation.outputs[0]] = self._run_operation(operation, values) + return values + + def run_outputs(self, inputs: list[np.ndarray]) -> list[np.ndarray]: + values = self.run(dict(zip(self.inputs, inputs))) + return [values[tensor_id] for tensor_id in self.outputs] + + def _run_operation(self, operation, values) -> np.ndarray: # noqa: C901 + tensors = self.graph.tensors + output = tensors[operation.outputs[0]] + opcode = operation.opcode + + if opcode in (XTS3OpCode.CONV2D, XTS3OpCode.DEPTHWISE_CONV2D): + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) + channels = output.shape[1] + p = parse_conv_params(operation.params, channels) + weights = self.constant(operation.inputs[1]) + bias = self.constant(operation.inputs[2]) + height, width, in_channels = x.shape + out_h, out_w = output.shape[2], output.shape[3] + dh, dw = p.dilation_h, p.dilation_w + padded = np.zeros( + (height + 2 * p.pad_h, width + 2 * p.pad_w, in_channels), dtype=np.int64 + ) + padded[p.pad_h : p.pad_h + height, p.pad_w : p.pad_w + width] = ( + x - source.quant_zero_point + ) + accumulator = np.zeros((out_h, out_w, channels), dtype=np.int64) + if opcode == XTS3OpCode.CONV2D: + groups = p.groups + group_in = in_channels // groups + group_out = channels // groups + filters = weights.reshape(channels, p.kernel_h, p.kernel_w, group_in) + for kh in range(p.kernel_h): + for kw in range(p.kernel_w): + patch = padded[ + kh * dh : kh * dh + + p.stride_h * (out_h - 1) + + 1 : p.stride_h, + kw * dw : kw * dw + + p.stride_w * (out_w - 1) + + 1 : p.stride_w, + ] + for g in range(groups): + accumulator[ + :, :, g * group_out : (g + 1) * group_out + ] += _exact_matmul( + patch[:, :, g * group_in : (g + 1) * group_in].reshape( + -1, group_in + ), + filters[ + g * group_out : (g + 1) * group_out, kh, kw, : + ].T, + ).reshape( + out_h, out_w, group_out + ) + else: + multiplier = channels // in_channels + filters = weights.reshape(p.kernel_h, p.kernel_w, channels) + expanded = ( + np.repeat(padded, multiplier, axis=2) if multiplier > 1 else padded + ) + for kh in range(p.kernel_h): + for kw in range(p.kernel_w): + patch = expanded[ + kh * dh : kh * dh + + p.stride_h * (out_h - 1) + + 1 : p.stride_h, + kw * dw : kw * dw + + p.stride_w * (out_w - 1) + + 1 : p.stride_w, + ] + accumulator += patch * filters[kh, kw] + accumulator = wrap_int32(accumulator + bias) + result = multiply_by_quantized_multiplier( + accumulator, p.multipliers, p.shifts + ) + result = result + output.quant_zero_point + return np.clip(result, p.act_min, p.act_max).astype(np.int8) + + if opcode == XTS3OpCode.RELU6: + act_min, act_max = struct.unpack(INT8_PAIR_PARAMS, operation.params) + return np.clip(values[operation.inputs[0]], act_min, act_max).astype( + np.int8 + ) + + if opcode == XTS3OpCode.ADD: + ( + lhs_mult, + lhs_shift, + rhs_mult, + rhs_shift, + left_shift, + out_mult, + out_shift, + act_min, + act_max, + ) = struct.unpack(INT8_ADD_PARAMS, operation.params) + lhs = tensors[operation.inputs[0]] + rhs = tensors[operation.inputs[1]] + a = ( + self.value(lhs.tensor_id, values).astype(np.int64) + - lhs.quant_zero_point + ) << left_shift + b = ( + self.value(rhs.tensor_id, values).astype(np.int64) + - rhs.quant_zero_point + ) << left_shift + a = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(a, lhs_mult), -lhs_shift + ) + b = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(b, rhs_mult), -rhs_shift + ) + total = wrap_int32(a + b) + total = rounding_divide_by_power_of_two( + saturating_rounding_doubling_high_mul(total, out_mult), -out_shift + ) + total = total + output.quant_zero_point + return np.clip(total, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.AVGPOOL2D: + multiplier, shift = struct.unpack(INT8_PAIR_PARAMS, operation.params) + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) + elements = x.shape[0] * x.shape[1] + total = x.sum(axis=(0, 1)) - elements * source.quant_zero_point + result = multiply_by_quantized_multiplier(total, multiplier, shift) + result = result + output.quant_zero_point + return np.clip(result, -128, 127).astype(np.int8).reshape(1, 1, -1) + + if opcode == XTS3OpCode.FLATTEN: + source = tensors[operation.inputs[0]] + x = values[source.tensor_id] + if len(source.shape) == 4: + return np.ascontiguousarray(x.transpose(2, 0, 1)).reshape(-1) + return x.reshape(-1) + + if opcode in (XTS3OpCode.MAX_POOL2D, XTS3OpCode.AVG_POOL2D): + kh, kw, sh, sw, ph, pw, _, _, act_min, act_max = struct.unpack( + INT8_POOL_PARAMS, operation.params + ) + x = values[operation.inputs[0]].astype(np.int64) + height, width, channels = x.shape + out_h, out_w = output.shape[2], output.shape[3] + result = np.zeros((out_h, out_w, channels), dtype=np.int64) + for oy in range(out_h): + y0 = oy * sh - ph + ys, ye = max(0, y0), min(height, y0 + kh) + for ox in range(out_w): + x0 = ox * sw - pw + xs, xe = max(0, x0), min(width, x0 + kw) + window = x[ys:ye, xs:xe].reshape(-1, channels) + if opcode == XTS3OpCode.MAX_POOL2D: + result[oy, ox] = window.max(axis=0) + else: + count = window.shape[0] + total = window.sum(axis=0) + half = count // 2 + numerator = np.where(total > 0, total + half, total - half) + result[oy, ox] = np.sign(numerator) * ( + np.abs(numerator) // count + ) + return np.clip(result, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.LUT: + table = np.frombuffer(operation.params, dtype=np.int8) + x = values[operation.inputs[0]].astype(np.int64) + return table[x + 128] + + if opcode == XTS3OpCode.MUL: + multiplier, shift, act_min, act_max, mode = struct.unpack( + INT8_MUL_PARAMS, operation.params + ) + lhs = tensors[operation.inputs[0]] + rhs = tensors[operation.inputs[1]] + a = ( + self.value(lhs.tensor_id, values).astype(np.int64) + - lhs.quant_zero_point + ) + b = ( + self.value(rhs.tensor_id, values).astype(np.int64) + - rhs.quant_zero_point + ) + if mode == MUL_MODE_CHANNEL_BROADCAST: + b = b.reshape(-1) + product = multiply_by_quantized_multiplier(a * b, multiplier, shift) + product = product + output.quant_zero_point + return np.clip(product, act_min, act_max).astype(np.int8) + + if opcode == XTS3OpCode.SOFTMAX: + table = np.frombuffer(operation.params, dtype="> ( + 2 * UPSAMPLE_WEIGHT_BITS + ) + return np.clip(result, -128, 127).astype(np.int8) + + if opcode == XTS3OpCode.TRANSPOSE_CONV2D: + source = tensors[operation.inputs[0]] + x = values[source.tensor_id].astype(np.int64) - source.quant_zero_point + channels = output.shape[1] + fields = struct.unpack_from(TRANSPOSE_CONV_PARAMS, operation.params, 0) + kh, kw, sh, sw, ph, pw, _, _, dh, dw, _, act_min, act_max = fields + base = TRANSPOSE_CONV_PARAMS_SIZE + multipliers = np.frombuffer( + operation.params, dtype="= 0) & (ox < out_w) + if not np.any(valid): + continue + contribution = _exact_matmul( + x[iy][valid], filters[:, ky, kx, :].T + ) + accumulator[oy, ox[valid]] += contribution + accumulator = wrap_int32(accumulator + bias) + result = multiply_by_quantized_multiplier(accumulator, multipliers, shifts) + result = result + output.quant_zero_point + return np.clip(result, act_min, act_max).astype(np.int8) + + raise ValueError(f"Unsupported opcode {opcode}") + + +def plan_activation_arena( + graph: XTS3Graph, alignment: int = 16 +) -> tuple[int, dict[int, int]]: + """Mirror of Int8Interpreter::plan_memory in the firmware.""" + op_count = len(graph.operations) + big = 1 << 30 + first: dict[int, int] = {} + last: dict[int, int] = {} + sizes: dict[int, int] = {} + for tensor in graph.tensors: + if tensor.is_constant: + continue + first[tensor.tensor_id] = -1 if tensor.flags & TENSOR_FLAG_GRAPH_INPUT else big + last[tensor.tensor_id] = ( + op_count if tensor.flags & TENSOR_FLAG_GRAPH_OUTPUT else -1 + ) + size = int(np.prod(tensor.shape)) + sizes[tensor.tensor_id] = (size + alignment - 1) // alignment * alignment + for index, operation in enumerate(graph.operations): + for tensor_id in operation.inputs: + if tensor_id in last: + last[tensor_id] = max(last[tensor_id], index) + for tensor_id in operation.outputs: + first[tensor_id] = min(first[tensor_id], index) + last[tensor_id] = max(last[tensor_id], index) + order = sorted( + (tensor_id for tensor_id in sizes if first[tensor_id] != big), + key=lambda tensor_id: (-sizes[tensor_id], tensor_id), + ) + offsets: dict[int, int] = {} + arena = 0 + for tensor_id in order: + candidate = 0 + moved = True + while moved: + moved = False + for other, offset in offsets.items(): + if not ( + first[other] <= last[tensor_id] and first[tensor_id] <= last[other] + ): + continue + if ( + candidate < offset + sizes[other] + and offset < candidate + sizes[tensor_id] + ): + candidate = offset + sizes[other] + moved = True + offsets[tensor_id] = candidate + arena = max(arena, candidate + sizes[tensor_id]) + return arena, offsets + + +__all__ = [ + "ReferenceInterpreter", + "plan_activation_arena", + "dequantize", + "device_shape", + "multiply_by_quantized_multiplier", + "quantize", + "rounding_divide_by_power_of_two", + "saturating_rounding_doubling_high_mul", + "to_device_layout", + "to_logical_layout", +] diff --git a/backends/espressif/reference/program_emulator.py b/backends/espressif/reference/program_emulator.py new file mode 100644 index 00000000000..98754ab76b2 --- /dev/null +++ b/backends/espressif/reference/program_emulator.py @@ -0,0 +1,98 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import numpy as np +import torch + +from ..serialization.xts3 import parse_graph + +from . import esp_nn_reference as reference + + +def _is_dequantize(node) -> bool: + return node.op == "call_function" and "dequantize_per_tensor" in str(node.target) + + +def delegate_output_dtypes(lowered_module) -> list[torch.dtype]: + program = lowered_module.original_module + output = next(node for node in program.graph.nodes if node.op == "output") + values = output.args[0] + if not isinstance(values, (list, tuple)): + values = (values,) + return [torch.float32 if _is_dequantize(value) else torch.int8 for value in values] + + +class DelegateEmulator: + """Runs one ESP32 int8 delegate payload exactly as the firmware backend does.""" + + def __init__(self, lowered_module): + self.graph = parse_graph(bytes(lowered_module.processed_bytes)) + self.interpreter = reference.ReferenceInterpreter(self.graph) + self.output_dtypes = delegate_output_dtypes(lowered_module) + + def __call__(self, *inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: + device_inputs = [] + for tensor_id, value in zip(self.interpreter.inputs, inputs): + tensor = self.graph.tensors[tensor_id] + array = value.detach().cpu().numpy() + if value.dtype == torch.float32: + array = reference.quantize(tensor, array) + elif value.dtype != torch.int8: + raise TypeError( + f"Delegate input must be float32 or int8, got {value.dtype}" + ) + device_inputs.append( + reference.to_device_layout(tensor, array.astype(np.int8)) + ) + outputs = self.interpreter.run_outputs(device_inputs) + results = [] + for tensor_id, value, dtype in zip( + self.interpreter.outputs, outputs, self.output_dtypes + ): + tensor = self.graph.tensors[tensor_id] + logical = reference.to_logical_layout(tensor, value) + if dtype == torch.float32: + results.append(torch.from_numpy(reference.dequantize(tensor, logical))) + else: + results.append(torch.from_numpy(logical.astype(np.int8))) + return tuple(results) + + +class ProgramEmulator(torch.fx.Interpreter): + """ + Host model of an ExecuTorch program lowered with EspNNPartitioner: + delegates run on the bit-exact ESP-NN reference, every other operator + runs in PyTorch (the device uses the ExecuTorch portable kernels). + """ + + def __init__(self, exported_program): + module = exported_program.module() + super().__init__(module) + self.delegates: dict[str, DelegateEmulator] = {} + self.delegate_calls = 0 + self.cpu_calls = 0 + + def call_function(self, target, args, kwargs): + if "executorch_call_delegate" in str(target): + lowered = args[0] + key = id(lowered) + if key not in self.delegates: + self.delegates[key] = DelegateEmulator(lowered) + self.delegate_calls += 1 + return self.delegates[key](*args[1:]) + if "getitem" not in str(target): + self.cpu_calls += 1 + return super().call_function(target, args, kwargs) + + def __call__(self, *inputs): + return self.run(*inputs) + + +__all__ = ["DelegateEmulator", "ProgramEmulator", "delegate_output_dtypes"] diff --git a/backends/espressif/serialization/__init__.py b/backends/espressif/serialization/__init__.py new file mode 100644 index 00000000000..b3a94f6980b --- /dev/null +++ b/backends/espressif/serialization/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .xts3 import build_graph, parse_graph, serialize_graph, XTS3_VERSION + +__all__ = ["XTS3_VERSION", "build_graph", "parse_graph", "serialize_graph"] diff --git a/backends/espressif/serialization/int8_ops.py b/backends/espressif/serialization/int8_ops.py new file mode 100644 index 00000000000..6dc75184020 --- /dev/null +++ b/backends/espressif/serialization/int8_ops.py @@ -0,0 +1,689 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import math +import struct + +import torch + +from .xts3 import ( + _activation_range, + _fusable_activation, + _normalize_int_tuple, + _per_channel_weight_scales, + _quantize_multiplier, + _require_output_qparams, + _serialize_per_channel_requantization, + _shape_from_node, + _target_kind, + COPY_FLAG_FILL, + COPY_HEADER, + COPY_MAX_RANK, + COPY_SEGMENT, + TensorRegistry, + TRANSPOSE_CONV_PARAMS, + UPSAMPLE_HEADER, + UPSAMPLE_MODE_BILINEAR, + UPSAMPLE_MODE_NEAREST, + UPSAMPLE_WEIGHT_ONE, + XTS3DType, + XTS3OpCode, + XTS3Operation, + XTS3Tensor, +) + + +MOVEMENT_KINDS = { + "reshape", + "flatten", + "unflatten", + "permute", + "transpose", + "squeeze", + "unsqueeze", + "slice", + "select", + "expand", + "split_item", + "identity_move", +} + +SPLIT_KINDS = {"split", "split_with_sizes", "unbind", "chunk"} + + +def is_dequantize(node) -> bool: + return ( + hasattr(node, "op") + and node.op == "call_function" + and "dequantize_per_tensor" in str(node.target) + ) + + +def is_quantize(node) -> bool: + return ( + hasattr(node, "op") + and node.op == "call_function" + and "quantize_per_tensor" in str(node.target) + and "dequantize" not in str(node.target) + ) + + +def node_kind(node) -> str | None: + if not hasattr(node, "op") or node.op != "call_function": + return None + kind = _target_kind(node.target) + if ( + kind == "getitem" + and _target_kind(getattr(node.args[0], "target", None)) in SPLIT_KINDS + ): + return "split_item" + if kind in ("identity", "dropout") and node.args: + source = node.args[0] + while is_dequantize(source) and is_quantize(source.args[0]): + source = source.args[0].args[0] + if node_kind(source) in MOVEMENT_KINDS: + return "identity_move" + return kind + + +def _skip_quantization(node): + while is_dequantize(node) and is_quantize(node.args[0]): + node = node.args[0].args[0] + while node_kind(node) in ("identity", "dropout"): + node = node.args[0] + while is_dequantize(node) and is_quantize(node.args[0]): + node = node.args[0].args[0] + return node + + +def consumers(node) -> list: + result = [] + for user in node.users: + if is_quantize(user): + for dq in user.users: + if is_dequantize(dq): + result.extend(consumers(dq)) + else: + result.append(dq) + elif node_kind(user) in ("identity", "dropout"): + result.extend(consumers(user)) + else: + result.append(user) + return result + + +def is_movement_terminal(node) -> bool: + users = consumers(node) + if not users: + return True + return any(node_kind(user) not in MOVEMENT_KINDS for user in users) + + +def _movement_chain(node): + chain = [] + current = node + while node_kind(current) in MOVEMENT_KINDS: + chain.append(current) + if node_kind(current) == "split_item": + current = _skip_quantization(current.args[0].args[0]) + else: + current = _skip_quantization(current.args[0]) + chain.reverse() + return current, chain + + +def _logical_index_view(shape: tuple[int, ...]) -> torch.Tensor: + count = max(1, math.prod(shape)) + base = torch.arange(count, dtype=torch.int64) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError(f"Rank-4 int8 tensors must have batch 1, got {shape}") + return base.view(shape[2], shape[3], shape[1]).permute(2, 0, 1).unsqueeze(0) + return base.view(shape) + + +def _device_view(view: torch.Tensor) -> torch.Tensor: + if view.dim() == 4: + if view.shape[0] != 1: + raise ValueError( + f"Rank-4 int8 tensors must have batch 1, got {tuple(view.shape)}" + ) + return view[0].permute(1, 2, 0) + return view + + +def _resolved_shape(view: torch.Tensor, shape) -> list[int]: + shape = [int(dim) for dim in shape] + if shape.count(-1) == 1: + known = math.prod(dim for dim in shape if dim != -1) + shape[shape.index(-1)] = view.numel() // max(known, 1) + return shape + + +def _split_bounds(split_node, index: int) -> tuple[int, int, int, bool]: + kind = _target_kind(split_node.target) + source_shape = _shape_from_node(split_node.args[0]) + if kind == "unbind": + dim = int(split_node.args[1]) if len(split_node.args) > 1 else 0 + dim = dim % len(source_shape) + return dim, index, index + 1, True + dim = ( + int(split_node.args[2]) + if len(split_node.args) > 2 + else int(split_node.kwargs.get("dim", 0)) + ) + dim = dim % len(source_shape) + size = source_shape[dim] + if kind == "chunk": + chunks = int(split_node.args[1]) + step = (size + chunks - 1) // chunks + sizes = [min(step, size - start) for start in range(0, size, step)] + elif kind == "split_with_sizes" or isinstance(split_node.args[1], (list, tuple)): + sizes = [int(v) for v in split_node.args[1]] + else: + step = int(split_node.args[1]) + sizes = [min(step, size - start) for start in range(0, size, step)] + start = sum(sizes[:index]) + return dim, start, start + sizes[index], False + + +def _apply_movement(view: torch.Tensor, node): # noqa: C901 + kind = node_kind(node) + args = node.args + if kind in ("reshape", "flatten", "unflatten"): + if kind == "reshape": + shape = _resolved_shape(view, args[1]) + else: + shape = list(_shape_from_node(node)) + try: + return view.view(shape), None + except RuntimeError: + return None, shape + if kind == "permute": + return view.permute([int(d) for d in args[1]]), None + if kind == "transpose": + return view.transpose(int(args[1]), int(args[2])), None + if kind == "squeeze": + if len(args) > 1: + dims = args[1] if isinstance(args[1], (list, tuple)) else [args[1]] + return view.squeeze(tuple(int(d) for d in dims)), None + return view.squeeze(), None + if kind == "unsqueeze": + return view.unsqueeze(int(args[1])), None + if kind == "slice": + dim = int(args[1]) if len(args) > 1 else 0 + start = args[2] if len(args) > 2 else None + end = args[3] if len(args) > 3 else None + step = int(args[4]) if len(args) > 4 else 1 + return torch.ops.aten.slice.Tensor(view, dim, start, end, step), None + if kind == "select": + return view.select(int(args[1]), int(args[2])), None + if kind == "expand": + return view.expand([int(d) for d in args[1]]), None + if kind == "identity_move": + return view, None + if kind == "split_item": + dim, start, end, squeeze = _split_bounds(node.args[0], int(node.args[1])) + sliced = torch.ops.aten.slice.Tensor(view, dim, start, end, 1) + return (sliced.squeeze(dim) if squeeze else sliced), None + raise ValueError(f"Unsupported data movement {node.name}") + + +def _coalesce(dims, in_strides, out_strides): + items = [ + (int(d), int(i), int(o)) + for d, i, o in zip(dims, in_strides, out_strides) + if int(d) != 1 + ] + merged = [] + for dim, in_stride, out_stride in items: + if merged: + p_dim, p_in, p_out = merged[-1] + if p_in == in_stride * dim and p_out == out_stride * dim: + merged[-1] = (p_dim * dim, in_stride, out_stride) + continue + merged.append((dim, in_stride, out_stride)) + if not merged: + merged = [(1, 1, 1)] + return [m[0] for m in merged], [m[1] for m in merged], [m[2] for m in merged] + + +def _contiguous_strides(dims) -> list[int]: + strides = [] + running = 1 + for dim in reversed(dims): + strides.append(running) + running *= int(dim) + return list(reversed(strides)) + + +def copy_segment( + input_index: int, + dims, + in_strides, + in_offset: int, + out_strides, + out_offset: int, + source: XTS3Tensor, + result: XTS3Tensor, +) -> bytes: + dims, in_strides, out_strides = _coalesce(dims, in_strides, out_strides) + if len(dims) > COPY_MAX_RANK: + raise ValueError( + f"Data movement needs {len(dims)} dimensions; the limit is {COPY_MAX_RANK}" + ) + requant = ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ) + multiplier, shift = (0, 0) + if requant: + multiplier, shift = _quantize_multiplier( + source.quant_scale / result.quant_scale + ) + pad = COPY_MAX_RANK - len(dims) + return struct.pack( + COPY_SEGMENT, + input_index, + len(dims), + int(in_offset), + int(out_offset), + *(list(dims) + [1] * pad), + *(list(in_strides) + [0] * pad), + *(list(out_strides) + [0] * pad), + int(requant), + multiplier, + shift, + source.quant_zero_point, + result.quant_zero_point, + ) + + +def copy_operation(inputs, output_id, segments, fill_value=None) -> XTS3Operation: + flags = COPY_FLAG_FILL if fill_value is not None else 0 + header = struct.pack(COPY_HEADER, len(segments), flags, int(fill_value or 0)) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.COPY, + inputs=list(inputs), + outputs=[output_id], + params=header + b"".join(segments), + ) + + +def _gather_operation(source_id, output_id, view, registry): + source = registry.tensors[source_id] + result = registry.tensors[output_id] + device = _device_view(view) + segment = copy_segment( + 0, + list(device.shape), + list(device.stride()), + device.storage_offset(), + _contiguous_strides(device.shape), + 0, + source, + result, + ) + return copy_operation([source_id], output_id, [segment]) + + +def _new_int8_tensor(registry: TensorRegistry, shape, like: XTS3Tensor) -> int: + tensor_id = len(registry.tensors) + registry.tensors.append( + XTS3Tensor( + tensor_id=tensor_id, + shape=tuple(int(d) for d in shape), + dtype=XTS3DType.INT8, + quant_scale=like.quant_scale, + quant_zero_point=like.quant_zero_point, + ) + ) + return tensor_id + + +def lower_movement( + node, registry: TensorRegistry, extra_operations: list +) -> XTS3Operation | None: + if not is_movement_terminal(node): + registry.fused_nodes.add(node) + return None + _require_output_qparams(registry, node) + source_node, chain = _movement_chain(node) + source_id = registry.get_or_create(source_node) + view = _logical_index_view(registry.tensors[source_id].shape) + current_id = source_id + for link in chain: + moved, reshape_to = _apply_movement(view, link) + if moved is None: + temp_id = _new_int8_tensor( + registry, (view.numel(),), registry.tensors[current_id] + ) + temp = registry.tensors[temp_id] + segment = copy_segment( + 0, + list(view.shape), + list(view.stride()), + view.storage_offset(), + _contiguous_strides(view.shape), + 0, + registry.tensors[current_id], + temp, + ) + extra_operations.append(copy_operation([current_id], temp_id, [segment])) + current_id = temp_id + view = torch.arange(view.numel(), dtype=torch.int64).view(reshape_to) + else: + view = moved + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + if tuple(view.shape) != tuple(result.shape): + raise ValueError( + f"Data movement {node.name} traced to {tuple(view.shape)}, expected {result.shape}" + ) + return _gather_operation(current_id, output_id, view, registry) + + +def lower_cat( + node, registry: TensorRegistry, extra_operations: list, stack: bool = False +) -> XTS3Operation: + sources = list(node.args[0]) + dim = int(node.args[1]) if len(node.args) > 1 else int(node.kwargs.get("dim", 0)) + _require_output_qparams(registry, node) + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + rank = len(result.shape) + dim = dim % rank + out_view = _logical_index_view(result.shape) + input_ids = [registry.get_or_create(source) for source in sources] + operations = [] + offset = 0 + pending_inputs = [] + pending_segments = [] + for input_id in input_ids: + source = registry.tensors[input_id] + if stack: + if len(source.shape) != rank - 1: + raise ValueError(f"stack {node.name} mixes ranks") + part = out_view.select(dim, offset) + offset += 1 + else: + if len(source.shape) != rank: + raise ValueError(f"cat {node.name} mixes ranks") + size = source.shape[dim] + part = torch.ops.aten.slice.Tensor(out_view, dim, offset, offset + size, 1) + offset += size + if source.shape and math.prod(source.shape) == 0: + continue + in_device = _device_view(_logical_index_view(source.shape)) + out_device = _device_view(part) + dims = list(in_device.shape) + if len(pending_inputs) == 6: + operations.append( + copy_operation(pending_inputs, output_id, pending_segments) + ) + pending_inputs, pending_segments = [], [] + pending_segments.append( + copy_segment( + len(pending_inputs), + dims, + list(in_device.stride()), + in_device.storage_offset(), + list(out_device.stride()), + out_device.storage_offset(), + source, + result, + ) + ) + pending_inputs.append(input_id) + operations.append(copy_operation(pending_inputs, output_id, pending_segments)) + extra_operations.extend(operations[:-1]) + return operations[-1] + + +def lower_pad(node, registry: TensorRegistry) -> XTS3Operation: + pads = [int(v) for v in node.args[1]] + value = ( + float(node.args[2]) + if len(node.args) > 2 + else float(node.kwargs.get("value", 0.0)) + ) + if any(p < 0 for p in pads): + raise ValueError(f"Negative padding (cropping) is not supported: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + rank = len(result.shape) + out_view = _logical_index_view(result.shape) + region = out_view + for i in range(len(pads) // 2): + dim = rank - 1 - i + before = pads[2 * i] + region = torch.ops.aten.slice.Tensor( + region, dim, before, before + source.shape[dim], 1 + ) + in_device = _device_view(_logical_index_view(source.shape)) + out_device = _device_view(region) + fill = round(value / result.quant_scale) + result.quant_zero_point + fill = max(-128, min(127, fill)) + segment = copy_segment( + 0, + list(in_device.shape), + list(in_device.stride()), + 0, + list(out_device.stride()), + out_device.storage_offset(), + source, + result, + ) + return copy_operation([input_id], output_id, [segment], fill_value=fill) + + +def broadcast_operand( + arg, target_shape, registry: TensorRegistry, extra_operations: list +) -> int: + tensor_id = registry.get_or_create(arg) + tensor = registry.tensors[tensor_id] + if tuple(tensor.shape) == tuple(target_shape): + return tensor_id + if tensor.is_constant: + return registry.broadcast_constant(tensor_id, tuple(target_shape)) + shape = (1,) * (len(target_shape) - len(tensor.shape)) + tuple(tensor.shape) + view = _logical_index_view(tensor.shape).reshape(shape).expand(*target_shape) + temp_id = _new_int8_tensor(registry, target_shape, tensor) + extra_operations.append(_gather_operation(tensor_id, temp_id, view, registry)) + return temp_id + + +def upsample_tables(node, in_size: int, out_size: int, axis: int, bilinear: bool): + args = node.args + kind = _target_kind(node.target) + scales = None + align_corners = False + if bilinear: + align_corners = ( + bool(args[2]) + if len(args) > 2 + else bool(node.kwargs.get("align_corners", False)) + ) + if kind.endswith("_vec"): + scales = args[3] if len(args) > 3 else node.kwargs.get("scale_factors") + else: + factor = args[3 + axis] if len(args) > 3 + axis else None + scales = None if factor is None else [factor, factor] + else: + if kind.endswith("_vec"): + scales = args[2] if len(args) > 2 else node.kwargs.get("scale_factors") + else: + factor = args[2 + axis] if len(args) > 2 + axis else None + scales = None if factor is None else [factor, factor] + scale_factor = None + if scales is not None and len(scales) == 2 and scales[axis] is not None: + scale_factor = float(scales[axis]) + table = [] + if not bilinear: + ratio = (1.0 / scale_factor) if scale_factor else in_size / out_size + for dst in range(out_size): + src = min(int(math.floor(dst * ratio)), in_size - 1) + table.extend([src, src, 0]) + return table + if align_corners: + ratio = (in_size - 1) / (out_size - 1) if out_size > 1 else 0.0 + else: + ratio = (1.0 / scale_factor) if scale_factor else in_size / out_size + for dst in range(out_size): + if align_corners: + src = dst * ratio + else: + src = max((dst + 0.5) * ratio - 0.5, 0.0) + x0 = min(int(math.floor(src)), in_size - 1) + x1 = x0 + 1 if x0 < in_size - 1 else x0 + weight = int(round((src - x0) * UPSAMPLE_WEIGHT_ONE)) if x1 != x0 else 0 + table.extend([x0, x1, weight]) + return table + + +def lower_upsample(node, registry: TensorRegistry) -> XTS3Operation: + kind = _target_kind(node.target) + bilinear = kind.startswith("upsample_bilinear2d") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if len(source.shape) != 4 or source.shape[0] != 1: + raise ValueError(f"Upsampling needs a batch-1 NCHW tensor: {node.name}") + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError(f"Upsampling {node.name} must keep its input quantization") + in_h, in_w = source.shape[2], source.shape[3] + out_h, out_w = result.shape[2], result.shape[3] + y_table = upsample_tables(node, in_h, out_h, 0, bilinear) + x_table = upsample_tables(node, in_w, out_w, 1, bilinear) + params = ( + struct.pack( + UPSAMPLE_HEADER, + UPSAMPLE_MODE_BILINEAR if bilinear else UPSAMPLE_MODE_NEAREST, + in_h, + in_w, + out_h, + out_w, + ) + + struct.pack(f"<{len(y_table)}i", *y_table) + + struct.pack(f"<{len(x_table)}i", *x_table) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.UPSAMPLE, + inputs=[input_id], + outputs=[output_id], + params=params, + ) + + +def transposed_conv_args(node) -> dict: + kind = _target_kind(node.target) + args = node.args + if kind == "convolution": + return { + "input": args[0], + "weight": args[1], + "bias": args[2], + "stride": _normalize_int_tuple(args[3], 2, "stride"), + "padding": _normalize_int_tuple(args[4], 2, "padding"), + "dilation": _normalize_int_tuple(args[5], 2, "dilation"), + "output_padding": _normalize_int_tuple(args[7], 2, "output_padding"), + "groups": int(args[8]), + } + return { + "input": args[0], + "weight": args[1], + "bias": args[2] if len(args) > 2 else None, + "stride": _normalize_int_tuple(args[3] if len(args) > 3 else 1, 2, "stride"), + "padding": _normalize_int_tuple(args[4] if len(args) > 4 else 0, 2, "padding"), + "output_padding": _normalize_int_tuple( + args[5] if len(args) > 5 else 0, 2, "output_padding" + ), + "groups": int(args[6]) if len(args) > 6 else 1, + "dilation": _normalize_int_tuple( + args[7] if len(args) > 7 else 1, 2, "dilation" + ), + } + + +def lower_transposed_conv(node, registry: TensorRegistry) -> XTS3Operation: + conv = transposed_conv_args(node) + if conv["groups"] != 1: + raise ValueError( + f"Grouped transposed convolution is not supported: {node.name}" + ) + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_id = registry.get_or_create(conv["input"]) + weight_scales = _per_channel_weight_scales(registry, conv["weight"]) + weight_id = registry.get_or_create(conv["weight"], "TRANSPOSED") + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + conv["bias"], input_tensor.quant_scale, weight_scales + ) + output_id = registry.get_or_create(output_node) + output_tensor = registry.tensors[output_id] + if activation is not None: + registry.fused_nodes.add(activation) + weight_shape = _shape_from_node(conv["weight"]) + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack( + TRANSPOSE_CONV_PARAMS, + weight_shape[2], + weight_shape[3], + *conv["stride"], + *conv["padding"], + *conv["output_padding"], + *conv["dilation"], + conv["groups"], + act_min, + act_max, + ) + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.TRANSPOSE_CONV2D, + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + +__all__ = [ + "MOVEMENT_KINDS", + "SPLIT_KINDS", + "broadcast_operand", + "copy_operation", + "copy_segment", + "is_dequantize", + "is_movement_terminal", + "is_quantize", + "lower_cat", + "lower_movement", + "lower_pad", + "lower_transposed_conv", + "lower_upsample", + "node_kind", + "transposed_conv_args", + "upsample_tables", +] diff --git a/backends/espressif/serialization/xts3.py b/backends/espressif/serialization/xts3.py new file mode 100644 index 00000000000..0a99476b05e --- /dev/null +++ b/backends/espressif/serialization/xts3.py @@ -0,0 +1,2140 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import struct +from dataclasses import dataclass, field +from enum import IntEnum +from math import frexp, prod + +import torch + + +MAGIC = b"XTS3" +XTS3_VERSION = 6 +XTS3_VERSIONS = (5, 6) +IO_TABLE_HEADER = " int: + if dtype == XTS3DType.FLOAT32: + return 4 + if dtype == XTS3DType.INT8: + return 1 + if dtype == XTS3DType.UINT8: + return 1 + if dtype == XTS3DType.INT32: + return 4 + raise ValueError(f"Unsupported ESP32-S3 dtype: {dtype}") + + +TENSOR_FLAG_CONSTANT = 1 << 0 +TENSOR_FLAG_QUANTIZED = 1 << 1 +TENSOR_FLAG_GRAPH_INPUT = 1 << 2 +TENSOR_FLAG_GRAPH_OUTPUT = 1 << 3 + + +@dataclass +class XTS3Tensor: + tensor_id: int + shape: tuple[int, ...] + dtype: XTS3DType + data_offset: int = 0 + data_size: int = 0 + flags: int = 0 + quant_scale: float = 0.0 + quant_zero_point: int = 0 + + @property + def is_constant(self) -> bool: + return bool(self.flags & TENSOR_FLAG_CONSTANT) + + +@dataclass +class XTS3Operation: + op_id: int + opcode: XTS3OpCode + inputs: list[int] = field(default_factory=list) + outputs: list[int] = field(default_factory=list) + params: bytes = b"" + + +@dataclass +class XTS3Graph: + tensors: list[XTS3Tensor] = field(default_factory=list) + operations: list[XTS3Operation] = field(default_factory=list) + constant_data: bytes = b"" + version: int = XTS3_VERSION + tensor_names: dict[int, str] = field(default_factory=dict) + inputs: list[int] = field(default_factory=list) + outputs: list[int] = field(default_factory=list) + + +class ExportedTensorResolver: + def __init__(self, exported_program): + self.exported_program = exported_program + self.graph_signature = getattr( + exported_program, + "graph_signature", + None, + ) + self.state_dict = ( + getattr( + exported_program, + "state_dict", + {}, + ) + or {} + ) + + def _lookup_mapping(self, mapping, name): + if mapping is None: + return None + + if name in mapping: + return mapping[name] + + for key, value in mapping.items(): + if str(key) == str(name): + return value + + return None + + def resolve(self, node): # noqa: C901 + if node is not None and node.op == "call_function": + target = str(node.target) + if "dequantize_per_channel" in target: + quantized = self.resolve(node.args[0]) + scales = self.resolve(node.args[1]) + zero_points = self.resolve(node.args[2]) + axis = int(node.args[3]) + if quantized is None or scales is None or zero_points is None: + return None + view = [1] * quantized.dim() + view[axis] = -1 + scale_values = scales.to(torch.float32).reshape(view) + zero_values = zero_points.to(torch.float32).reshape(view) + return (quantized.to(torch.float32) - zero_values) * scale_values + if node.op != "placeholder": + return None + + name = node.name + target = node.target + + parameter_mapping = getattr( + self.graph_signature, + "inputs_to_parameters", + {}, + ) + + buffer_mapping = getattr( + self.graph_signature, + "inputs_to_buffers", + {}, + ) + + state_name = self._lookup_mapping( + parameter_mapping, + name, + ) + + if state_name is None: + state_name = self._lookup_mapping( + parameter_mapping, + target, + ) + + if state_name is None: + state_name = self._lookup_mapping( + buffer_mapping, + name, + ) + + if state_name is None: + state_name = self._lookup_mapping( + buffer_mapping, + target, + ) + + if state_name is not None and state_name in self.state_dict: + return self.state_dict[state_name] + + constants = getattr(self.exported_program, "constants", None) or {} + constant_mapping = getattr( + self.graph_signature, + "inputs_to_lifted_tensor_constants", + {}, + ) + constant_name = self._lookup_mapping(constant_mapping, name) + if constant_name is None: + constant_name = self._lookup_mapping(constant_mapping, target) + if constant_name is not None and constant_name in constants: + return constants[constant_name] + + for candidate in (target, name): + if candidate in self.state_dict: + return self.state_dict[candidate] + if candidate in constants: + return constants[candidate] + + return None + + +def _shape_from_node(node) -> tuple[int, ...]: + value = node.meta.get("val") + + if value is None: + return () + + shape = getattr( + value, + "shape", + None, + ) + + if shape is None: + return () + + return tuple(int(dim) for dim in shape) + + +def _dtype_from_node(node) -> torch.dtype: + value = node.meta.get("val") + + dtype = getattr( + value, + "dtype", + None, + ) + + if dtype is None: + raise ValueError(f"Node {node.name} has no dtype metadata") + + return dtype + + +def _normalize_int_tuple( + value, + length: int, + name: str, +) -> tuple[int, ...]: + + if isinstance(value, int): + return (int(value),) * length + + result = tuple(int(x) for x in value) + + if len(result) != length: + raise ValueError(f"Expected {length} values for " f"{name}, got {result}") + + return result + + +def _target_text(target) -> str: + return str(target) + + +LUT_KINDS = ( + "hardswish", + "hardsigmoid", + "sigmoid", + "silu", + "tanh", + "gelu", + "elu", + "leaky_relu", +) + + +def _target_kind(target) -> str | None: # noqa: C901 + text = _target_text(target) + + if "aten.convolution.default" in text: + return "convolution" + + if "aten.conv_transpose2d" in text: + return "conv_transpose2d" + + if "aten.upsample_nearest2d.vec" in text: + return "upsample_nearest2d_vec" + + if "aten.upsample_nearest2d.default" in text: + return "upsample_nearest2d" + + if "aten.upsample_bilinear2d.vec" in text: + return "upsample_bilinear2d_vec" + + if "aten.upsample_bilinear2d.default" in text: + return "upsample_bilinear2d" + + if "aten.constant_pad_nd.default" in text or "aten.pad.default" in text: + return "pad" + + for name, kind in ( + ("permute", "permute"), + ("transpose", "transpose"), + ("unsqueeze", "unsqueeze"), + ("squeeze", "squeeze"), + ("expand", "expand"), + ("unflatten", "unflatten"), + ("split_with_sizes", "split_with_sizes"), + ("unbind", "unbind"), + ("chunk", "chunk"), + ): + if f"aten.{name}." in text or f"aten.{name}_copy." in text: + return kind + + if "aten.split." in text or "aten.split_copy." in text: + return "split" + + if "aten.slice.Tensor" in text or "aten.slice_copy.Tensor" in text: + return "slice" + + if "aten.select.int" in text or "aten.select_copy.int" in text: + return "select" + + if "aten.conv2d.default" in text: + return "conv2d" + + if "aten.hardtanh_.default" in text: + return "hardtanh_" + + if "aten.hardtanh.default" in text: + return "hardtanh" + + if "aten.relu_.default" in text or "aten.relu.default" in text: + return "relu" + + if "aten.relu6.default" in text: + return "relu6" + + if "aten.add.Tensor" in text or "aten.add_.Tensor" in text: + return "add" + + if "aten.mul.Tensor" in text or "aten.mul_.Tensor" in text: + return "mul" + + if "aten._softmax.default" in text or "aten.softmax.int" in text: + return "softmax" + + for name in LUT_KINDS: + if f"aten.{name}.default" in text or f"aten.{name}_.default" in text: + return name + + if "aten.adaptive_avg_pool2d.default" in text: + return "adaptive_avg_pool2d" + + if "aten.max_pool2d_with_indices.default" in text: + return "max_pool2d_with_indices" + + if "aten.max_pool2d.default" in text: + return "max_pool2d" + + if "aten.avg_pool2d.default" in text: + return "avg_pool2d" + + if "aten.cat.default" in text: + return "cat" + + if "aten.stack.default" in text: + return "stack" + + if any( + marker in text + for marker in ( + "aten.clone.default", + "_clone_dim_order.default", + "aten.alias.default", + "aten.alias_copy.default", + "aten.detach.default", + "aten.detach_copy.default", + ) + ): + return "identity" + + if "aten.dropout.default" in text or "aten.dropout_.default" in text: + return "dropout" + + if text == "" or text.endswith("operator.getitem"): + return "getitem" + + if any( + marker in text + for marker in ( + "aten.view.default", + "aten.view_copy.default", + "aten.reshape.default", + "aten._unsafe_view.default", + ) + ): + return "reshape" + + if "aten.mean.dim" in text: + return "mean_dim" + + if "aten.flatten.using_ints" in text: + return "flatten" + + if "aten.linear.default" in text: + return "linear" + + return None + + +def _int8_device_bytes(values: torch.Tensor, shape: tuple[int, ...]) -> bytes: + values = values.to(torch.int8).reshape(shape) + if len(shape) == 4: + if shape[0] != 1: + raise ValueError(f"Rank-4 int8 constants must have batch 1, got {shape}") + values = values[0].permute(1, 2, 0) + return values.contiguous().numpy().tobytes() + + +def _int8_from_device(values: torch.Tensor, shape: tuple[int, ...]) -> torch.Tensor: + if len(shape) == 4: + return ( + values.reshape(shape[2], shape[3], shape[1]).permute(2, 0, 1).unsqueeze(0) + ) + return values.reshape(shape) + + +class TensorRegistry: + def __init__(self, resolver: ExportedTensorResolver, graph_module): + self.resolver = resolver + self.tensors: list[XTS3Tensor] = [] + self.node_to_id: dict[object, int] = {} + self.constant_data = bytearray() + self.fused_nodes: set = set() + self.extra_operations: list = [] + self.node_to_qparams: dict[object, tuple[float, int]] = {} + for node in graph_module.graph.nodes: + if node.op != "call_function" or "quantize_per_tensor" not in str( + node.target + ): + continue + if len(node.args) < 3 or not hasattr(node.args[0], "meta"): + continue + self.node_to_qparams[node.args[0]] = ( + float(node.args[1]), + int(node.args[2]), + ) + + def _align_constant_data(self, alignment: int = 16) -> None: + self.constant_data.extend(b"\x00" * ((-len(self.constant_data)) % alignment)) + + def get_or_create(self, node, filter_layout: str | None = None) -> int: + return self._get_or_create_int8(node, filter_layout) + + def _append_constant(self, tensor: XTS3Tensor, raw: bytes) -> int: + self._align_constant_data() + tensor.data_offset = len(self.constant_data) + tensor.data_size = len(raw) + tensor.flags |= TENSOR_FLAG_CONSTANT + self.constant_data.extend(raw) + self.tensors.append(tensor) + return tensor.tensor_id + + def _new_int8_activation(self, node, scale: float, zero_point: int) -> int: + if node in self.node_to_id: + tensor_id = self.node_to_id[node] + tensor = self.tensors[tensor_id] + if tensor.quant_scale != scale or tensor.quant_zero_point != zero_point: + raise ValueError( + f"Tensor {node.name} has inconsistent activation qparams" + ) + return tensor_id + shape = _shape_from_node(node) + if not shape: + raise ValueError(f"Quantized activation {node.name} has no shape metadata") + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + quant_scale=float(scale), + quant_zero_point=int(zero_point), + ) + self.tensors.append(tensor) + self.node_to_id[node] = tensor_id + return tensor_id + + def _constant_int8(self, node) -> int: + quantized_node = node.args[0] + if quantized_node in self.node_to_id: + return self.node_to_id[quantized_node] + quantized = self.resolver.resolve(quantized_node) + if quantized is None or quantized.dtype != torch.int8: + raise ValueError( + f"Expected quantize_per_tensor or an int8 constant before {node.name}" + ) + shape = tuple(int(dim) for dim in quantized.shape) + if len(shape) > 4: + raise ValueError( + f"Constant {node.name} has rank {len(shape)}; the limit is 4" + ) + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + quant_scale=float(node.args[1]), + quant_zero_point=int(node.args[2]), + ) + self._append_constant( + tensor, _int8_device_bytes(quantized.detach().cpu(), shape) + ) + self.node_to_id[quantized_node] = tensor_id + return tensor_id + + def broadcast_constant(self, tensor_id: int, shape: tuple[int, ...]) -> int: + source = self.tensors[tensor_id] + values = torch.frombuffer( + bytearray( + self.constant_data[ + source.data_offset : source.data_offset + source.data_size + ] + ), + dtype=torch.int8, + ) + logical = _int8_from_device(values, source.shape) + expanded = ( + logical.reshape( + (1,) * (len(shape) - len(source.shape)) + tuple(source.shape) + ) + .expand(shape) + .contiguous() + ) + result_id = len(self.tensors) + result = XTS3Tensor( + tensor_id=result_id, + shape=tuple(shape), + dtype=XTS3DType.INT8, + quant_scale=source.quant_scale, + quant_zero_point=source.quant_zero_point, + ) + self._append_constant(result, _int8_device_bytes(expanded, tuple(shape))) + return result_id + + def _get_or_create_int8( # noqa: C901 + self, node, filter_layout: str | None = None + ) -> int: + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized_node = node.args[0] + if not ( + getattr(quantized_node, "op", None) == "call_function" + and "quantize_per_tensor" in str(quantized_node.target) + ): + if ( + quantized_node.op == "placeholder" + and self.resolver.resolve(quantized_node) is None + ): + return self._new_int8_activation( + quantized_node, float(node.args[1]), int(node.args[2]) + ) + return self._constant_int8(node) + source = quantized_node.args[0] + if getattr( + source, "op", None + ) == "call_function" and "dequantize_per_tensor" in str(source.target): + # q(dq(x)) is left behind where an identity op (such as an + # eval-mode dropout) was decomposed away. + if (float(source.args[1]), int(source.args[2])) != ( + float(node.args[1]), + int(node.args[2]), + ): + raise ValueError( + f"{quantized_node.name} requantizes {source.name}; " + "back-to-back requantization is not supported" + ) + return self._get_or_create_int8(source, filter_layout) + return self._new_int8_activation( + source, + float(node.args[1]), + int(node.args[2]), + ) + + if node.op == "call_function" and "dequantize_per_channel" in str(node.target): + quantized_node = node.args[0] + if quantized_node in self.node_to_id: + return self.node_to_id[quantized_node] + quantized = self.resolver.resolve(quantized_node) + if quantized is None or quantized.dtype != torch.int8: + raise ValueError( + f"Quantized weight {node.name} is not backed by int8 data" + ) + expected_axis = 1 if filter_layout == "TRANSPOSED" else 0 + if int(node.args[3]) != expected_axis: + raise ValueError( + f"Only output-channel weight quantization is supported: {node.name}" + ) + if not torch.all(self.resolver.resolve(node.args[2]) == 0): + raise ValueError( + f"Only symmetric int8 weights are supported: {node.name}" + ) + shape = tuple(int(dim) for dim in quantized.shape) + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=XTS3DType.INT8, + ) + raw = quantized.detach().cpu().contiguous() + if len(shape) == 4 and filter_layout == "OHWI": + raw = raw.permute(0, 2, 3, 1).contiguous() + elif len(shape) == 4 and filter_layout == "TRANSPOSED": + raw = raw.permute(1, 2, 3, 0).contiguous() + tensor.shape = (shape[1], shape[2], shape[3], shape[0]) + elif len(shape) == 4 and filter_layout == "HWOC": + raw = ( + raw.permute(2, 3, 0, 1) + .reshape(shape[2], shape[3], shape[0]) + .contiguous() + ) + elif len(shape) == 4: + raise ValueError( + f"A convolution filter layout is required for {node.name}" + ) + raw_bytes = raw.numpy().tobytes() + self._append_constant(tensor, raw_bytes) + self.node_to_id[quantized_node] = tensor_id + return tensor_id + + if node in self.node_to_id: + return self.node_to_id[node] + + quantization = self.node_to_qparams.get(node) + if quantization is None: + raise ValueError( + f"Activation {node.name} has no PT2E quantization parameters" + ) + return self._new_int8_activation(node, *quantization) + + def add_int32_bias( + self, node, input_scale: float, weight_scales: torch.Tensor + ) -> int: + scales = weight_scales.detach().cpu().to(torch.float64).reshape(-1) + if node is None: + bias = torch.zeros(scales.numel(), dtype=torch.float64) + else: + bias = self.resolver.resolve(node) + if bias is None: + raise ValueError(f"Unable to resolve bias tensor {node.name}") + bias = bias.detach().cpu().to(torch.float64).reshape(-1) + if bias.numel() != scales.numel(): + raise ValueError("Bias and per-channel weight scale counts differ") + bias_q = torch.round(bias / (float(input_scale) * scales)) + if torch.any(bias_q < -(1 << 31)) or torch.any(bias_q > (1 << 31) - 1): + raise ValueError("Quantized bias overflows int32") + tensor_id = len(self.tensors) + tensor = XTS3Tensor( + tensor_id=tensor_id, + shape=(int(bias_q.numel()),), + dtype=XTS3DType.INT32, + ) + raw_bytes = bias_q.to(torch.int32).contiguous().numpy().tobytes() + self._append_constant(tensor, raw_bytes) + return tensor_id + + +def _get_conv_args(node): + args = node.args + + if len(args) < 2: + raise ValueError( + f"Convolution node {node.name} " f"has fewer than two arguments" + ) + + kind = _target_kind(node.target) + + if kind == "convolution": + if len(args) < 9: + raise ValueError( + f"Edge convolution node " + f"{node.name} has unexpected " + f"argument count: {len(args)}" + ) + + transposed = bool(args[6]) + + output_padding = _normalize_int_tuple( + args[7], + 2, + "output_padding", + ) + + if transposed: + raise ValueError( + f"Transposed convolution must use the transposed lowering: {node.name}" + ) + + if any(output_padding): + raise ValueError( + f"Non-zero output_padding is " f"not supported: {node.name}" + ) + + return { + "input": args[0], + "weight": args[1], + "bias": args[2], + "stride": _normalize_int_tuple( + args[3], + 2, + "stride", + ), + "padding": _normalize_int_tuple( + args[4], + 2, + "padding", + ), + "dilation": _normalize_int_tuple( + args[5], + 2, + "dilation", + ), + "groups": int(args[8]), + } + + return { + "input": args[0], + "weight": args[1], + "bias": (args[2] if len(args) > 2 else None), + "stride": _normalize_int_tuple( + args[3] if len(args) > 3 else (1, 1), + 2, + "stride", + ), + "padding": _normalize_int_tuple( + args[4] if len(args) > 4 else (0, 0), + 2, + "padding", + ), + "dilation": _normalize_int_tuple( + args[5] if len(args) > 5 else (1, 1), + 2, + "dilation", + ), + "groups": (int(args[6]) if len(args) > 6 else 1), + } + + +def _is_depthwise_conv( + node, + conv_args, +) -> bool: + + groups = conv_args["groups"] + + weight_shape = _shape_from_node(conv_args["weight"]) + + input_shape = _shape_from_node(conv_args["input"]) + + if len(weight_shape) != 4 or len(input_shape) < 2: + return False + + return groups > 1 and groups == input_shape[1] and weight_shape[1] == 1 + + +def _is_relu6(node) -> bool: + if len(node.args) < 3: + return False + + try: + minimum = float(node.args[1]) + maximum = float(node.args[2]) + except ( + TypeError, + ValueError, + ): + return False + + return minimum == 0.0 and maximum == 6.0 + + +def _quantize_multiplier(real_multiplier: float) -> tuple[int, int]: + if not real_multiplier > 0.0: + raise ValueError( + f"Quantized multiplier must be positive, got {real_multiplier}" + ) + significand, exponent = frexp(real_multiplier) + multiplier = round(significand * (1 << 31)) + if multiplier == (1 << 31): + multiplier >>= 1 + exponent += 1 + if multiplier > 0x7FFFFFFF: + multiplier = 0x7FFFFFFF + if exponent < -31 or exponent > 30: + raise ValueError( + f"Requantization multiplier {real_multiplier} is outside the ESP-NN shift range" + ) + return int(multiplier), int(exponent) + + +def _quantize_multiplier_smaller_than_one(real_multiplier: float) -> tuple[int, int]: + if not 0.0 < real_multiplier < 1.0: + raise ValueError(f"Expected a multiplier in (0, 1), got {real_multiplier}") + multiplier, shift = _quantize_multiplier(real_multiplier) + if shift > 0: + raise ValueError(f"Unexpected positive shift for {real_multiplier}") + return multiplier, shift + + +def _per_channel_weight_scales(registry: TensorRegistry, weight_node) -> torch.Tensor: + if weight_node.op != "call_function" or "dequantize_per_channel" not in str( + weight_node.target + ): + raise ValueError( + f"Expected per-channel quantized weights at {weight_node.name}" + ) + scales = registry.resolver.resolve(weight_node.args[1]) + zero_points = registry.resolver.resolve(weight_node.args[2]) + if scales is None or zero_points is None: + raise ValueError( + f"Unable to resolve per-channel weight qparams for {weight_node.name}" + ) + scales = scales.detach().cpu().to(torch.float64).reshape(-1) + if not torch.all(torch.isfinite(scales)) or torch.any(scales <= 0): + raise ValueError(f"Invalid per-channel weight scales for {weight_node.name}") + if not torch.all(zero_points == 0): + raise ValueError( + f"Only symmetric per-channel weights are supported for {weight_node.name}" + ) + return scales + + +def _serialize_per_channel_requantization( + input_scale: float, + weight_scales: torch.Tensor, + output_scale: float, +) -> bytes: + multipliers = [] + shifts = [] + for weight_scale in weight_scales.tolist(): + multiplier, shift = _quantize_multiplier( + float(input_scale) * float(weight_scale) / float(output_scale) + ) + multipliers.append(multiplier) + shifts.append(shift) + return struct.pack(f"<{len(multipliers)}i", *multipliers) + struct.pack( + f"<{len(shifts)}i", *shifts + ) + + +def _round_half_away(value: float) -> int: + return int(value + 0.5) if value >= 0 else -int(-value + 0.5) + + +def _activation_range( + kind: str | None, scale: float, zero_point: int +) -> tuple[int, int]: + if kind is None: + return -128, 127 + low = max(-128, zero_point) + if kind == "relu": + return low, 127 + if kind == "relu6": + return low, min(127, zero_point + _round_half_away(6.0 / scale)) + raise ValueError(f"Unsupported fused activation {kind}") + + +def _int8_activation_kind(node) -> str | None: + kind = _target_kind(node.target) if node.op == "call_function" else None + if kind == "relu": + return "relu" + if kind == "relu6": + return "relu6" + if kind in ("hardtanh", "hardtanh_") and _is_relu6(node): + return "relu6" + return None + + +def _fusable_activation(node, registry: TensorRegistry): + users = list(node.users) + if len(users) != 1: + return None, None + activation = users[0] + kind = _int8_activation_kind(activation) + if kind is None or activation.args[0] is not node: + return None, None + if activation not in registry.node_to_qparams: + return None, None + return activation, kind + + +def _require_output_qparams(registry: TensorRegistry, node) -> None: + if node not in registry.node_to_qparams: + raise ValueError( + f"Int8 operation {node.name} has no output quantization; " + "quantize the model with EspNNQuantizer before lowering" + ) + + +def lower_int8_node( # noqa: C901 + node, registry: TensorRegistry +) -> XTS3Operation | None: + if node.op != "call_function" or node in registry.fused_nodes: + return None + kind = _target_kind(node.target) + + if kind == "conv_transpose2d" or ( + kind == "convolution" and len(node.args) > 6 and bool(node.args[6]) + ): + from .int8_ops import lower_transposed_conv + + return lower_transposed_conv(node, registry) + + if kind in ( + "upsample_nearest2d", + "upsample_nearest2d_vec", + "upsample_bilinear2d", + "upsample_bilinear2d_vec", + ): + from .int8_ops import lower_upsample + + return lower_upsample(node, registry) + + if kind == "pad": + from .int8_ops import lower_pad + + return lower_pad(node, registry) + + if kind in ("split", "split_with_sizes", "unbind", "chunk"): + registry.fused_nodes.add(node) + return None + + if kind in ( + "permute", + "transpose", + "squeeze", + "unsqueeze", + "slice", + "select", + "expand", + "unflatten", + ) or ( + kind == "getitem" + and _target_kind(getattr(node.args[0], "target", None)) + in ("split", "split_with_sizes", "unbind", "chunk") + ): + from .int8_ops import lower_movement + + return lower_movement(node, registry, registry.extra_operations) + + if kind in ("conv2d", "convolution"): + conv = _get_conv_args(node) + if ( + max(conv["dilation"]) > 255 + or max(conv["padding"]) > 255 + or max(conv["stride"]) > 255 + ): + raise ValueError( + f"Convolution geometry exceeds the 8-bit parameter fields: {node.name}" + ) + depthwise = _is_depthwise_conv(node, conv) + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_id = registry.get_or_create(conv["input"]) + weight_scales = _per_channel_weight_scales(registry, conv["weight"]) + weight_id = registry.get_or_create( + conv["weight"], "HWOC" if depthwise else "OHWI" + ) + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + conv["bias"], input_tensor.quant_scale, weight_scales + ) + output_id = registry.get_or_create(output_node) + output_tensor = registry.tensors[output_id] + if activation is not None: + registry.fused_nodes.add(activation) + weight_shape = _shape_from_node(conv["weight"]) + if len(weight_shape) != 4: + raise ValueError(f"Convolution weight must be rank 4: {node.name}") + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack( + INT8_CONV_BASE_PARAMS, + weight_shape[2], + weight_shape[3], + conv["stride"][0], + conv["stride"][1], + conv["padding"][0], + conv["padding"][1], + conv["dilation"][0], + conv["dilation"][1], + conv["groups"], + act_min, + act_max, + ) + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + return XTS3Operation( + op_id=-1, + opcode=(XTS3OpCode.DEPTHWISE_CONV2D if depthwise else XTS3OpCode.CONV2D), + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + if kind in ("hardtanh_", "hardtanh", "relu", "relu6"): + activation_kind = _int8_activation_kind(node) + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if activation_kind is None or ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + return _lut_operation(node, input_id, output_id, registry) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.RELU6, + inputs=[input_id], + outputs=[output_id], + params=struct.pack(INT8_PAIR_PARAMS, act_min, act_max), + ) + + if kind in LUT_KINDS: + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + return _lut_operation(node, input_id, output_id, registry) + + if kind in ("add", "mul"): + scalar_operation = _lower_scalar_binary(node, kind, registry) + if scalar_operation is not None: + return scalar_operation + + if kind == "mul": + return _lower_int8_mul(node, registry) + + if kind == "softmax": + return _lower_int8_softmax(node, registry) + + if kind == "add": + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + from .int8_ops import broadcast_operand + + output_id = registry.get_or_create(output_node) + result = registry.tensors[output_id] + lhs_id = broadcast_operand( + node.args[0], result.shape, registry, registry.extra_operations + ) + rhs_id = broadcast_operand( + node.args[1], result.shape, registry, registry.extra_operations + ) + if activation is not None: + registry.fused_nodes.add(activation) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + if lhs.shape != rhs.shape or lhs.shape != result.shape: + raise ValueError(f"Broadcasting add could not be resolved: {node.name}") + twice_max_input_scale = 2.0 * max(lhs.quant_scale, rhs.quant_scale) + lhs_mult, lhs_shift = _quantize_multiplier_smaller_than_one( + lhs.quant_scale / twice_max_input_scale + ) + rhs_mult, rhs_shift = _quantize_multiplier_smaller_than_one( + rhs.quant_scale / twice_max_input_scale + ) + out_mult, out_shift = _quantize_multiplier_smaller_than_one( + twice_max_input_scale / ((1 << ADD_LEFT_SHIFT) * result.quant_scale) + ) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.ADD, + inputs=[lhs_id, rhs_id], + outputs=[output_id], + params=struct.pack( + INT8_ADD_PARAMS, + lhs_mult, + lhs_shift, + rhs_mult, + rhs_shift, + ADD_LEFT_SHIFT, + out_mult, + out_shift, + act_min, + act_max, + ), + ) + + if kind in ("adaptive_avg_pool2d", "mean_dim"): + if kind == "adaptive_avg_pool2d": + output_size = ( + tuple(int(v) for v in node.args[1]) if len(node.args) > 1 else () + ) + if output_size != (1, 1): + return _lower_adaptive_avg_pool(node, output_size, registry) + else: + if ( + len(node.args) < 3 + or tuple(int(d) for d in node.args[1]) not in ((2, 3), (-2, -1)) + or not bool(node.args[2]) + ): + raise ValueError(f"Only spatial keepdim mean is supported: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if len(source.shape) != 4: + raise ValueError(f"Average pooling input must be rank 4: {node.name}") + elements = source.shape[2] * source.shape[3] + mult, shift = _quantize_multiplier( + source.quant_scale / (result.quant_scale * elements) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.AVGPOOL2D, + inputs=[input_id], + outputs=[output_id], + params=struct.pack(INT8_PAIR_PARAMS, mult, shift), + ) + + if kind in ("max_pool2d", "avg_pool2d"): + return _lower_int8_pool(node, kind, registry) + + if kind == "max_pool2d_with_indices": + users = list(node.users) + if any( + _target_kind(user.target) != "getitem" or user.args[1] != 0 + for user in users + ): + raise ValueError( + f"max_pool2d indices are used by {node.name}; only values are supported" + ) + if len(users) != 1: + raise ValueError( + f"max_pool2d_with_indices {node.name} must have one getitem user" + ) + operation = _lower_int8_pool(users[0], "max_pool2d", registry, source_node=node) + registry.fused_nodes.add(users[0]) + return operation + + if kind in ("identity", "dropout"): + from .int8_ops import lower_movement, node_kind + + if node_kind(node) == "identity_move": + return lower_movement(node, registry, registry.extra_operations) + + if kind in ("identity", "dropout", "getitem"): + if kind == "dropout" and len(node.args) > 2 and bool(node.args[2]): + raise ValueError(f"Training-mode dropout cannot be lowered: {node.name}") + if kind == "getitem": + raise ValueError(f"Unsupported tuple access {node.name}") + source_id = registry.get_or_create(node.args[0]) + quantization = registry.node_to_qparams.get(node) + tensor = registry.tensors[source_id] + if quantization is not None and quantization != ( + tensor.quant_scale, + tensor.quant_zero_point, + ): + raise ValueError(f"Identity op {node.name} changes quantization parameters") + registry.node_to_id[node] = source_id + registry.fused_nodes.add(node) + return None + + if kind == "stack": + from .int8_ops import lower_cat + + return lower_cat(node, registry, registry.extra_operations, stack=True) + + if kind == "cat": + try: + return _lower_int8_concat(node, registry) + except ValueError: + from .int8_ops import lower_cat + + return lower_cat(node, registry, registry.extra_operations) + + if kind in ("flatten", "reshape"): + from .int8_ops import ( + _skip_quantization, + is_movement_terminal, + lower_movement, + MOVEMENT_KINDS, + node_kind, + ) + + if not is_movement_terminal(node): + registry.fused_nodes.add(node) + return None + _require_output_qparams(registry, node) + producer = _skip_quantization(node.args[0]) + result_shape = _shape_from_node(node) + if ( + node_kind(producer) in MOVEMENT_KINDS + or len(result_shape) != 2 + or result_shape[0] != 1 + ): + return lower_movement(node, registry, registry.extra_operations) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + return lower_movement(node, registry, registry.extra_operations) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.FLATTEN, + inputs=[input_id], + outputs=[output_id], + ) + + if kind == "linear": + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + input_shape = _shape_from_node(node.args[0]) + if len(input_shape) == 4 or len(input_shape) < 1: + raise ValueError( + f"linear {node.name} needs a rank 1 to 3 input, got {input_shape}" + ) + input_id = registry.get_or_create(node.args[0]) + weight_scales = _per_channel_weight_scales(registry, node.args[1]) + weight_id = registry.get_or_create(node.args[1]) + input_tensor = registry.tensors[input_id] + bias_id = registry.add_int32_bias( + node.args[2] if len(node.args) > 2 else None, + input_tensor.quant_scale, + weight_scales, + ) + output_id = registry.get_or_create(output_node) + if activation is not None: + registry.fused_nodes.add(activation) + output_tensor = registry.tensors[output_id] + act_min, act_max = _activation_range( + activation_kind, output_tensor.quant_scale, output_tensor.quant_zero_point + ) + params = struct.pack(INT8_PAIR_PARAMS, act_min, act_max) + ( + _serialize_per_channel_requantization( + input_tensor.quant_scale, weight_scales, output_tensor.quant_scale + ) + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.LINEAR, + inputs=[input_id, weight_id, bias_id], + outputs=[output_id], + params=params, + ) + + return None + + +def _unary_function(node, kind: str): + import torch.nn.functional as F + + args = list(node.args) + kwargs = dict(node.kwargs) + if kind == "hardswish": + return F.hardswish + if kind == "hardsigmoid": + return F.hardsigmoid + if kind == "sigmoid": + return torch.sigmoid + if kind == "silu": + return F.silu + if kind == "tanh": + return torch.tanh + if kind == "gelu": + approximate = kwargs.get("approximate", args[1] if len(args) > 1 else "none") + return lambda x: F.gelu(x, approximate=approximate) + if kind == "elu": + alpha = float(args[1]) if len(args) > 1 else float(kwargs.get("alpha", 1.0)) + scale = float(args[2]) if len(args) > 2 else float(kwargs.get("scale", 1.0)) + input_scale = ( + float(args[3]) if len(args) > 3 else float(kwargs.get("input_scale", 1.0)) + ) + return lambda x: torch.ops.aten.elu(x, alpha, scale, input_scale) + if kind == "leaky_relu": + slope = ( + float(args[1]) + if len(args) > 1 + else float(kwargs.get("negative_slope", 0.01)) + ) + return lambda x: F.leaky_relu(x, slope) + if kind == "relu": + return torch.relu + if kind == "relu6": + return lambda x: torch.clamp(x, 0.0, 6.0) + if kind in ("hardtanh", "hardtanh_"): + low = float(args[1]) if len(args) > 1 else -1.0 + high = float(args[2]) if len(args) > 2 else 1.0 + return lambda x: torch.clamp(x, low, high) + raise ValueError(f"No lookup-table function for {node.name}") + + +def build_lut(function, source: XTS3Tensor, result: XTS3Tensor) -> bytes: + codes = torch.arange(-128, 128, dtype=torch.int32).to(torch.int8) + values = torch.ops.quantized_decomposed.dequantize_per_tensor( + codes, source.quant_scale, source.quant_zero_point, -128, 127, torch.int8 + ) + outputs = function(values.to(torch.float32)).to(torch.float32) + if not torch.all(torch.isfinite(outputs)): + raise ValueError("Lookup-table function produced non-finite values") + quantized = torch.ops.quantized_decomposed.quantize_per_tensor( + outputs, result.quant_scale, result.quant_zero_point, -128, 127, torch.int8 + ) + return quantized.contiguous().numpy().tobytes() + + +def _lut_operation( + node, input_id: int, output_id: int, registry: TensorRegistry, function=None +): + kind = _target_kind(node.target) + function = function or _unary_function(node, kind) + table = build_lut(function, registry.tensors[input_id], registry.tensors[output_id]) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.LUT, + inputs=[input_id], + outputs=[output_id], + params=table, + ) + + +def _constant_scalar(node, registry: TensorRegistry): + if isinstance(node, (int, float)): + return float(node) + if not hasattr(node, "op"): + return None + value = None + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized = registry.resolver.resolve(node.args[0]) + if quantized is not None: + value = torch.ops.quantized_decomposed.dequantize_per_tensor( + quantized, + float(node.args[1]), + int(node.args[2]), + int(node.args[3]), + int(node.args[4]), + quantized.dtype, + ) + elif node.op in ("placeholder", "get_attr"): + value = registry.resolver.resolve(node) + if value is None or value.numel() != 1: + return None + return float(value.reshape(-1)[0]) + + +def _reject_constant_operands(node, registry: TensorRegistry) -> None: + for arg in node.args[:2]: + if not hasattr(arg, "op"): + continue + source = ( + arg.args[0] + if (arg.op == "call_function" and "dequantize_per" in str(arg.target)) + else arg + ) + if hasattr(source, "op") and registry.resolver.resolve(source) is not None: + raise ValueError( + f"{node.name} uses a constant tensor operand; only scalar constants are supported" + ) + + +def _lower_scalar_binary(node, kind: str, registry: TensorRegistry): + lhs, rhs = node.args[0], node.args[1] + constant = _constant_scalar(rhs, registry) + tensor_arg = lhs + if constant is None: + constant = _constant_scalar(lhs, registry) + tensor_arg = rhs + if constant is None: + return None + alpha = float(node.kwargs.get("alpha", 1.0)) if kind == "add" else 1.0 + _require_output_qparams(registry, node) + input_id = registry.get_or_create(tensor_arg) + output_id = registry.get_or_create(node) + tensor_on_left = tensor_arg is lhs + + def function(x): + if kind != "add": + return x * constant + return x + constant * alpha if tensor_on_left else constant + alpha * x + + return _lut_operation(node, input_id, output_id, registry, function) + + +def _lower_int8_mul(node, registry: TensorRegistry) -> XTS3Operation: + activation, activation_kind = _fusable_activation(node, registry) + output_node = activation if activation is not None else node + _require_output_qparams(registry, output_node) + from .int8_ops import broadcast_operand + + lhs_id = registry.get_or_create(node.args[0]) + rhs_id = registry.get_or_create(node.args[1]) + output_id = registry.get_or_create(output_node) + if activation is not None: + registry.fused_nodes.add(activation) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + result = registry.tensors[output_id] + + def per_channel_pattern(full, small): + return ( + len(full.shape) == 4 + and len(small.shape) == 4 + and full.shape == result.shape + and small.shape == (1, full.shape[1], 1, 1) + and full.shape[0] == 1 + and not small.is_constant + ) + + if not (lhs.shape == rhs.shape == result.shape) and not ( + per_channel_pattern(lhs, rhs) or per_channel_pattern(rhs, lhs) + ): + lhs_id = broadcast_operand( + node.args[0], result.shape, registry, registry.extra_operations + ) + rhs_id = broadcast_operand( + node.args[1], result.shape, registry, registry.extra_operations + ) + lhs = registry.tensors[lhs_id] + rhs = registry.tensors[rhs_id] + if lhs.shape == rhs.shape == result.shape: + mode = MUL_MODE_ELEMENTWISE + inputs = [lhs_id, rhs_id] + else: + + def per_channel(full, small): + return ( + len(full.shape) == 4 + and len(small.shape) == 4 + and full.shape == result.shape + and small.shape == (1, full.shape[1], 1, 1) + and full.shape[0] == 1 + ) + + if per_channel(lhs, rhs): + inputs = [lhs_id, rhs_id] + elif per_channel(rhs, lhs): + inputs = [rhs_id, lhs_id] + else: + raise ValueError( + f"mul {node.name} broadcasts {lhs.shape} x {rhs.shape}; only identical shapes " + "or (1,C,H,W) x (1,C,1,1) are supported" + ) + mode = MUL_MODE_CHANNEL_BROADCAST + multiplier, shift = _quantize_multiplier( + lhs.quant_scale * rhs.quant_scale / result.quant_scale + ) + act_min, act_max = _activation_range( + activation_kind, result.quant_scale, result.quant_zero_point + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.MUL, + inputs=inputs, + outputs=[output_id], + params=struct.pack(INT8_MUL_PARAMS, multiplier, shift, act_min, act_max, mode), + ) + + +def build_softmax_table(input_scale: float) -> bytes: + import math + + one = 1 << SOFTMAX_EXP_BITS + table = [ + int(math.floor(math.exp(-d * input_scale) * one + 0.5)) for d in range(256) + ] + return struct.pack("<256I", *table) + + +def _lower_int8_softmax(node, registry: TensorRegistry) -> XTS3Operation: + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + rank = len(source.shape) + dim = int(node.args[1]) if len(node.args) > 1 else -1 + if dim < 0: + dim += rank + if source.shape[0] != 1 or not ( + (rank == 2 and dim == 1) or (rank == 4 and dim == 1) + ): + raise ValueError( + f"Only channel softmax on batch-1 rank 2 or 4 tensors is supported: {node.name}" + ) + if ( + abs(result.quant_scale - SOFTMAX_OUTPUT_SCALE) > 1e-9 + or result.quant_zero_point != SOFTMAX_OUTPUT_ZERO_POINT + ): + raise ValueError( + f"softmax {node.name} output must use scale 1/256 and zero point -128 " + "(quantize with EspNNQuantizer)" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.SOFTMAX, + inputs=[input_id], + outputs=[output_id], + params=build_softmax_table(source.quant_scale), + ) + + +def _pool_window(value, name: str) -> tuple[int, int]: + if value is None or (isinstance(value, (list, tuple)) and len(value) == 0): + return None + if isinstance(value, int): + return (int(value), int(value)) + values = tuple(int(v) for v in value) + if len(values) == 1: + return (values[0], values[0]) + if len(values) != 2: + raise ValueError(f"Expected a 2D {name}, got {values}") + return values + + +def _lower_int8_pool( + node, kind: str, registry: TensorRegistry, source_node=None +) -> XTS3Operation: + source_node = source_node if source_node is not None else node + args = list(source_node.args) + kernel = _pool_window(args[1], "kernel_size") + stride = _pool_window(args[2] if len(args) > 2 else None, "stride") or kernel + padding = _pool_window(args[3] if len(args) > 3 else 0, "padding") or (0, 0) + if kind == "max_pool2d": + dilation = _pool_window(args[4] if len(args) > 4 else 1, "dilation") or (1, 1) + ceil_mode = bool(args[5]) if len(args) > 5 else False + if dilation != (1, 1): + raise ValueError(f"Dilated max pooling is not supported: {node.name}") + opcode = XTS3OpCode.MAX_POOL2D + else: + ceil_mode = bool(args[4]) if len(args) > 4 else False + count_include_pad = bool(args[5]) if len(args) > 5 else True + divisor_override = args[6] if len(args) > 6 else None + if divisor_override is not None: + raise ValueError( + f"avg_pool2d divisor_override is not supported: {node.name}" + ) + if count_include_pad and padding != (0, 0): + raise ValueError( + f"avg_pool2d with padding must use count_include_pad=False for ESP-NN: {node.name}" + ) + opcode = XTS3OpCode.AVG_POOL2D + if ceil_mode: + raise ValueError(f"ceil_mode pooling is not supported: {node.name}") + if padding[0] * 2 > kernel[0] or padding[1] * 2 > kernel[1]: + raise ValueError(f"Pooling padding exceeds half the window: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(source_node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError(f"Pooling {node.name} changes quantization parameters") + if len(source.shape) != 4 or source.shape[0] != 1: + raise ValueError(f"Pooling input must be batch-1 rank 4: {node.name}") + return XTS3Operation( + op_id=-1, + opcode=opcode, + inputs=[input_id], + outputs=[output_id], + params=struct.pack( + INT8_POOL_PARAMS, + kernel[0], + kernel[1], + stride[0], + stride[1], + padding[0], + padding[1], + 1, + 1, + -128, + 127, + ), + ) + + +def _lower_adaptive_avg_pool( + node, output_size, registry: TensorRegistry +) -> XTS3Operation: + input_shape = _shape_from_node(node.args[0]) + if len(input_shape) != 4 or len(output_size) != 2: + raise ValueError(f"Adaptive pooling needs a rank-4 input: {node.name}") + height, width = input_shape[2], input_shape[3] + out_h, out_w = output_size + if height % out_h or width % out_w: + raise ValueError( + f"Adaptive pooling {node.name} maps {height}x{width} to {out_h}x{out_w}; " + "only evenly divisible sizes are supported" + ) + kernel = (height // out_h, width // out_w) + if max(kernel) > 255: + raise ValueError(f"Adaptive pooling window exceeds 255: {node.name}") + _require_output_qparams(registry, node) + input_id = registry.get_or_create(node.args[0]) + output_id = registry.get_or_create(node) + source = registry.tensors[input_id] + result = registry.tensors[output_id] + if ( + source.quant_scale != result.quant_scale + or source.quant_zero_point != result.quant_zero_point + ): + raise ValueError( + f"Adaptive pooling {node.name} changes quantization parameters" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.AVG_POOL2D, + inputs=[input_id], + outputs=[output_id], + params=struct.pack( + INT8_POOL_PARAMS, + kernel[0], + kernel[1], + kernel[0], + kernel[1], + 0, + 0, + 1, + 1, + -128, + 127, + ), + ) + + +def _lower_int8_concat(node, registry: TensorRegistry) -> XTS3Operation: + sources = list(node.args[0]) + dim = int(node.args[1]) if len(node.args) > 1 else 0 + if not 2 <= len(sources) <= 6: + raise ValueError(f"Concatenation needs 2 to 6 inputs: {node.name}") + _require_output_qparams(registry, node) + input_ids = [registry.get_or_create(source) for source in sources] + output_id = registry.get_or_create(node) + result = registry.tensors[output_id] + rank = len(result.shape) + if dim < 0: + dim += rank + if dim != 1 or rank not in (2, 4) or result.shape[0] != 1: + raise ValueError( + f"Only batch-1 channel concatenation is supported: {node.name}" + ) + for input_id in input_ids: + tensor = registry.tensors[input_id] + if ( + tensor.quant_scale != result.quant_scale + or tensor.quant_zero_point != result.quant_zero_point + ): + raise ValueError( + f"Concatenation inputs of {node.name} use different qparams" + ) + if len(tensor.shape) != rank or tensor.shape[2:] != result.shape[2:]: + raise ValueError( + f"Concatenation inputs of {node.name} have mismatched shapes" + ) + return XTS3Operation( + op_id=-1, + opcode=XTS3OpCode.CONCAT, + inputs=input_ids, + outputs=[output_id], + ) + + +INT8_PASSTHROUGH_MARKERS = ( + "quantize_per_tensor", + "dequantize_per_tensor", + "dequantize_per_channel", + "quantize_per_channel", +) + + +def _is_int8_passthrough(node) -> bool: + text = str(node.target) + return any(marker in text for marker in INT8_PASSTHROUGH_MARKERS) + + +def _quantized_source(node): + if node.op == "call_function" and "dequantize_per_tensor" in str(node.target): + quantized = node.args[0] + if getattr( + quantized, "op", None + ) == "call_function" and "quantize_per_tensor" in str(quantized.target): + return quantized.args[0] + if ( + node.op == "call_function" + and "quantize_per_tensor" in str(node.target) + and "dequantize" not in str(node.target) + ): + return node.args[0] + return None + + +def _user_input_placeholders(exported_program): + graph_module = exported_program.graph_module + placeholders = [ + node for node in graph_module.graph.nodes if node.op == "placeholder" + ] + signature = getattr(exported_program, "graph_signature", None) + if signature is None: + return placeholders + user_inputs = set(signature.user_inputs) + return [node for node in placeholders if node.name in user_inputs] + + +def _mark_int8_graph_io( + exported_program, registry: TensorRegistry +) -> tuple[list[int], list[int]]: + graph_module = exported_program.graph_module + inputs: list[int] = [] + for node in _user_input_placeholders(exported_program): + if node not in registry.node_to_id: + if registry.resolver.resolve(node) is not None: + continue + raise ValueError( + f"Graph input {node.name} is not consumed by an int8 operation" + ) + tensor = registry.tensors[registry.node_to_id[node]] + if tensor.is_constant: + continue + tensor.flags |= TENSOR_FLAG_GRAPH_INPUT + inputs.append(tensor.tensor_id) + if not inputs: + raise ValueError("Int8 graph has no quantized runtime input") + + outputs: list[int] = [] + for node in graph_module.graph.nodes: + if node.op != "output": + continue + values = node.args[0] + if not isinstance(values, (list, tuple)): + values = (values,) + for value in values: + source = _quantized_source(value) + if source is None or source not in registry.node_to_id: + raise ValueError( + f"Graph output {getattr(value, 'name', value)} is not a dequantized int8 tensor" + ) + registry.tensors[ + registry.node_to_id[source] + ].flags |= TENSOR_FLAG_GRAPH_OUTPUT + outputs.append(registry.node_to_id[source]) + if not outputs: + raise ValueError("Int8 graph has no outputs") + return inputs, outputs + + +def build_graph(exported_program) -> XTS3Graph: + """Lower a PT2E-quantized ExportedProgram (or delegate partition) to an XTS3 int8 graph.""" + graph_module = exported_program.graph_module + registry = TensorRegistry(ExportedTensorResolver(exported_program), graph_module) + operations = [] + for node in graph_module.graph.nodes: + if node.op != "call_function": + continue + lowered = lower_int8_node(node, registry) + for extra in registry.extra_operations: + extra.op_id = len(operations) + operations.append(extra) + registry.extra_operations.clear() + if lowered is None: + if node not in registry.fused_nodes and not _is_int8_passthrough(node): + raise ValueError( + f"Unsupported operation for the ESP-NN int8 backend: {node.name} ({node.target})" + ) + continue + lowered.op_id = len(operations) + operations.append(lowered) + graph_inputs, graph_outputs = _mark_int8_graph_io(exported_program, registry) + return XTS3Graph( + tensors=registry.tensors, + operations=operations, + constant_data=bytes(registry.constant_data), + version=XTS3_VERSION, + tensor_names={ + tensor_id: node.name for node, tensor_id in registry.node_to_id.items() + }, + inputs=graph_inputs, + outputs=graph_outputs, + ) + + +def _serialize_tensor(tensor: XTS3Tensor) -> bytes: + if len(tensor.shape) > 4: + raise ValueError(f"Tensor rank exceeds the XTS3 limit of 4: {tensor.shape}") + dims = list(tensor.shape) + [0] * (4 - len(tensor.shape)) + flags = tensor.flags | (TENSOR_FLAG_QUANTIZED if tensor.quant_scale > 0 else 0) + return struct.pack( + TENSOR_FORMAT, + tensor.tensor_id, + tensor.data_offset, + int(tensor.dtype), + flags, + len(tensor.shape), + *dims, + tensor.quant_scale, + tensor.quant_zero_point, + ) + + +def _serialize_operation(operation: XTS3Operation, params_offset: int) -> bytes: + if len(operation.inputs) > 6: + raise ValueError(f"Operation {operation.op_id} has more than 6 inputs") + if len(operation.outputs) > 2: + raise ValueError(f"Operation {operation.op_id} has more than 2 outputs") + inputs = operation.inputs + [0] * (6 - len(operation.inputs)) + outputs = operation.outputs + [0] * (2 - len(operation.outputs)) + return struct.pack( + OP_FORMAT, + operation.op_id & 0xFF, + int(operation.opcode), + len(operation.inputs), + len(operation.outputs), + *inputs, + *outputs, + params_offset, + len(operation.params), + ) + + +def check_runtime_limits(graph: XTS3Graph) -> None: + """ + Raise ValueError for graphs the device runtime would reject at init(): + the same limits espnn_runtime/src/xts3_graph.cpp enforces. + """ + if graph.version not in XTS3_VERSIONS: + raise ValueError(f"Cannot serialize XTS3 version {graph.version}") + if len(graph.inputs) > MAX_GRAPH_IO or len(graph.outputs) > MAX_GRAPH_IO: + raise ValueError( + f"Partition has {len(graph.inputs)} inputs and {len(graph.outputs)} " + f"outputs; the ESP-NN runtime supports at most {MAX_GRAPH_IO} of each" + ) + if len(graph.tensors) > MAX_GRAPH_TENSORS: + raise ValueError( + f"Partition has {len(graph.tensors)} tensors; the ESP-NN runtime " + f"supports at most {MAX_GRAPH_TENSORS}" + ) + if len(graph.operations) > MAX_GRAPH_OPERATIONS: + raise ValueError( + f"Partition has {len(graph.operations)} operations; the ESP-NN " + f"runtime supports at most {MAX_GRAPH_OPERATIONS}" + ) + newest = XTS3OpCode.SOFTMAX if graph.version == 5 else max(XTS3OpCode) + for operation in graph.operations: + if int(operation.opcode) > int(newest): + raise ValueError( + f"{XTS3OpCode(operation.opcode).name} needs XTS3 version 6, " + f"the graph is version {graph.version}" + ) + if operation.opcode == XTS3OpCode.UPSAMPLE: + _, _, _, out_h, out_w = struct.unpack_from( + UPSAMPLE_HEADER, operation.params + ) + if max(out_h, out_w) > UPSAMPLE_MAX_EXTENT: + raise ValueError( + f"Upsample output {out_h}x{out_w} exceeds " + f"{UPSAMPLE_MAX_EXTENT} in one dimension" + ) + for tensor in graph.tensors: + if ( + tensor.flags & TENSOR_FLAG_CONSTANT + and tensor.dtype == XTS3DType.INT32 + and tensor.data_offset % 4 + ): + raise ValueError(f"int32 constant {tensor.tensor_id} is not 4-byte aligned") + + +def check_dataflow(graph: XTS3Graph) -> None: + """ + Raise ValueError when an operation or graph output reads an activation + that is neither a graph input nor written by an earlier operation. The + device would read whatever its arena holds there. + """ + ready = set(graph.inputs) + ready.update( + tensor.tensor_id + for tensor in graph.tensors + if tensor.flags & TENSOR_FLAG_CONSTANT + ) + for index, operation in enumerate(graph.operations): + for tensor_id in operation.inputs: + if tensor_id >= 0 and tensor_id not in ready: + raise ValueError( + f"Operation {index} ({XTS3OpCode(operation.opcode).name}) " + f"reads tensor {tensor_id}, which nothing writes before it" + ) + ready.update(operation.outputs) + for tensor_id in graph.outputs: + if tensor_id not in ready: + raise ValueError(f"Graph output {tensor_id} is never written") + + +def serialize_graph(graph: XTS3Graph) -> bytes: + """Serialize to the XTS3 wire format; constants start 16-byte aligned.""" + check_runtime_limits(graph) + check_dataflow(graph) + tensor_data = b"".join(_serialize_tensor(tensor) for tensor in graph.tensors) + parameter_data = bytearray() + operation_data = bytearray() + for operation in graph.operations: + operation_data += _serialize_operation(operation, len(parameter_data)) + parameter_data += operation.params + io_table = b"" + if graph.version >= 6: + if len(graph.inputs) > 0xFFFF or len(graph.outputs) > 0xFFFF: + raise ValueError("Too many graph inputs or outputs") + io_table = struct.pack( + IO_TABLE_HEADER, len(graph.inputs), len(graph.outputs) + ) + struct.pack( + f"<{len(graph.inputs) + len(graph.outputs)}I", *graph.inputs, *graph.outputs + ) + prefix = ( + HEADER_SIZE + + len(io_table) + + len(tensor_data) + + len(operation_data) + + len(parameter_data) + ) + parameter_data += b"\x00" * ((-prefix) % 16) + header = struct.pack( + HEADER_FORMAT, + MAGIC, + graph.version, + len(graph.tensors), + len(graph.operations), + len(parameter_data), + len(graph.constant_data), + ) + return ( + header + + io_table + + tensor_data + + bytes(operation_data) + + bytes(parameter_data) + + bytes(graph.constant_data) + ) + + +def parse_graph(data: bytes) -> XTS3Graph: # noqa: C901 + """Parse and bounds-check an XTS3 v5 or v6 int8 graph.""" + if len(data) < HEADER_SIZE: + raise ValueError("Graph blob is smaller than the header") + magic, version, tensor_count, operation_count, parameter_size, constant_size = ( + struct.unpack_from(HEADER_FORMAT, data, 0) + ) + if magic != MAGIC: + raise ValueError(f"Invalid graph magic: {magic!r}") + if version not in XTS3_VERSIONS: + raise ValueError( + f"Unsupported XTS3 version {version}; expected one of {XTS3_VERSIONS}" + ) + offset = HEADER_SIZE + graph_inputs: list[int] = [] + graph_outputs: list[int] = [] + if version >= 6: + if offset + IO_TABLE_HEADER_SIZE > len(data): + raise ValueError("Graph IO table exceeds blob") + input_count, output_count = struct.unpack_from(IO_TABLE_HEADER, data, offset) + offset += IO_TABLE_HEADER_SIZE + if offset + 4 * (input_count + output_count) > len(data): + raise ValueError("Graph IO table exceeds blob") + ids = struct.unpack_from(f"<{input_count + output_count}I", data, offset) + graph_inputs, graph_outputs = list(ids[:input_count]), list(ids[input_count:]) + offset += 4 * (input_count + output_count) + + if offset + tensor_count * TENSOR_SIZE > len(data): + raise ValueError("Graph tensor region exceeds blob") + tensors = [] + for _ in range(tensor_count): + fields = struct.unpack_from(TENSOR_FORMAT, data, offset) + offset += TENSOR_SIZE + tensor_id, data_offset, dtype_id, flags, rank = fields[:5] + quant_scale, quant_zero_point = fields[9:11] + if rank > 4: + raise ValueError(f"Tensor {tensor_id} has invalid rank {rank}") + try: + dtype = XTS3DType(dtype_id) + except ValueError as exc: + raise ValueError( + f"Tensor {tensor_id} has invalid dtype id {dtype_id}" + ) from exc + shape = tuple(fields[5 : 5 + rank]) + tensors.append( + XTS3Tensor( + tensor_id=tensor_id, + shape=shape, + dtype=dtype, + data_offset=data_offset, + data_size=(prod(shape) if shape else 1) * dtype_size(dtype), + flags=flags, + quant_scale=quant_scale, + quant_zero_point=quant_zero_point, + ) + ) + + if offset + operation_count * OP_SIZE > len(data): + raise ValueError("Graph operation region exceeds blob") + raw_operations = [] + for _ in range(operation_count): + fields = struct.unpack_from(OP_FORMAT, data, offset) + offset += OP_SIZE + op_id, opcode, num_inputs, num_outputs = fields[:4] + params_offset, params_size = fields[12:14] + if num_inputs > 6 or num_outputs > 2: + raise ValueError(f"Operation {op_id} has invalid input or output counts") + if params_offset + params_size > parameter_size: + raise ValueError( + f"Operation {op_id} parameters exceed the parameter region" + ) + raw_operations.append( + ( + op_id, + XTS3OpCode(opcode), + list(fields[4:10])[:num_inputs], + list(fields[10:12])[:num_outputs], + params_offset, + params_size, + ) + ) + parameter_base = offset + constant_base = parameter_base + parameter_size + if constant_base + constant_size > len(data): + raise ValueError("Graph constant region exceeds blob") + operations = [ + XTS3Operation( + op_id=op_id, + opcode=opcode, + inputs=inputs, + outputs=outputs, + params=data[ + parameter_base + + params_offset : parameter_base + + params_offset + + params_size + ], + ) + for op_id, opcode, inputs, outputs, params_offset, params_size in raw_operations + ] + if version == 5: + graph_inputs = [ + t.tensor_id for t in tensors if t.flags & TENSOR_FLAG_GRAPH_INPUT + ] + graph_outputs = [ + t.tensor_id for t in tensors if t.flags & TENSOR_FLAG_GRAPH_OUTPUT + ] + for tensor_id in graph_inputs + graph_outputs: + if tensor_id >= len(tensors): + raise ValueError(f"Graph IO table references missing tensor {tensor_id}") + return XTS3Graph( + tensors=tensors, + operations=operations, + constant_data=data[constant_base : constant_base + constant_size], + version=version, + inputs=graph_inputs, + outputs=graph_outputs, + ) + + +__all__ = [ + "MAGIC", + "check_runtime_limits", + "XTS3_VERSION", + "XTS3_VERSIONS", + "XTS3DType", + "XTS3Graph", + "XTS3OpCode", + "XTS3Operation", + "XTS3Tensor", + "build_graph", + "lower_int8_node", + "parse_graph", + "serialize_graph", +] diff --git a/backends/espressif/test/__init__.py b/backends/espressif/test/__init__.py new file mode 100644 index 00000000000..2d8dd1da1a1 --- /dev/null +++ b/backends/espressif/test/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause diff --git a/backends/espressif/test/host/xts3_parse_main.cpp b/backends/espressif/test/host/xts3_parse_main.cpp new file mode 100644 index 00000000000..9fb02db1b7f --- /dev/null +++ b/backends/espressif/test/host/xts3_parse_main.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include + +int main(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::FILE* file = std::fopen(argv[i], "rb"); + if (file == nullptr) { + std::printf("%s missing\n", argv[i]); + return 2; + } + std::vector data; + uint8_t chunk[4096]; + size_t read = 0; + while ((read = std::fread(chunk, 1, sizeof(chunk), file)) > 0) { + data.insert(data.end(), chunk, chunk + read); + } + std::fclose(file); + executorch::backends::espressif::GraphView graph{}; + const bool accepted = executorch::backends::espressif::parse_graph( + data.data(), data.size(), graph); + std::printf("VERDICT %s %s\n", argv[i], accepted ? "accept" : "reject"); + } + return 0; +} diff --git a/backends/espressif/test/test_contracts.py b/backends/espressif/test/test_contracts.py new file mode 100644 index 00000000000..4d68ae301d2 --- /dev/null +++ b/backends/espressif/test/test_contracts.py @@ -0,0 +1,226 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Contract tests for the partitioner, quantizer, preprocess and serializer: +strict delegation, per-module quantization configs, and the device runtime +limits that must be caught on the host. + + pytest backends/espressif/test/test_contracts.py +""" + +from __future__ import annotations + +import dataclasses + +import pytest + +pytest.importorskip("torchao") + +import torch # noqa: E402 +import torch.nn as nn # noqa: E402 +from executorch.backends.espressif.espnn_preprocess import ( # noqa: E402 + EspNNBackend, + INT8_FORMAT_SPEC, +) +from executorch.backends.espressif.lowering import to_espnn_edge # noqa: E402 +from executorch.backends.espressif.partition import espnn_partitioner # noqa: E402 +from executorch.backends.espressif.quantizer.espnn_quantizer import ( # noqa: E402 + EspNNQuantizer, + fused_activation_kind, +) +from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 +from executorch.backends.espressif.serialization.xts3 import ( # noqa: E402 + build_graph, + check_dataflow, + check_runtime_limits, + MAX_GRAPH_IO, + MAX_GRAPH_OPERATIONS, + TENSOR_FLAG_CONSTANT, + XTS3DType, + XTS3OpCode, +) +from executorch.backends.espressif.test.test_int8_ops import quantize # noqa: E402 +from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( # noqa: E402 + get_symmetric_quantization_config, +) +from executorch.exir.backend.compile_spec_schema import CompileSpec # noqa: E402 +from torchao.quantization.pt2e.quantize_pt2e import prepare_pt2e # noqa: E402 + + +class FeaturesAndClassifier(nn.Module): + def __init__(self): + super().__init__() + self.features = nn.Sequential(nn.Conv2d(3, 8, 3, padding=1), nn.ReLU()) + self.classifier = nn.Sequential(nn.Conv2d(8, 4, 1), nn.ReLU()) + + def forward(self, x): + return self.classifier(self.features(x)) + + +def _small_graph(): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + return build_graph(program) + + +def test_strict_mode_raises_when_a_region_stays_on_cpu(monkeypatch): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + monkeypatch.setattr( + espnn_partitioner, "_boundary_is_valid", lambda nodes, resolver: False + ) + with pytest.raises(RuntimeError, match="would run on the CPU"): + to_espnn_edge(program, allow_cpu_fallback=False) + + +def test_fallback_mode_reports_rejected_regions(monkeypatch): + _, program = quantize(FeaturesAndClassifier(), [(1, 3, 8, 8)]) + monkeypatch.setattr( + espnn_partitioner, "_boundary_is_valid", lambda nodes, resolver: False + ) + _, report = to_espnn_edge(program) + assert report["partitions"] == 0 + assert report["rejected_partitions"] >= 1 + assert report["cpu_ops"] + + +def _weight_observer(prepared, conv_name_fragment): + modules = dict(prepared.named_modules()) + for node in prepared.graph.nodes: + if node.op != "call_function" or "conv" not in str(node.target): + continue + stack = node.meta.get("nn_module_stack", {}) + if not any(conv_name_fragment in path for path, _ in stack.values()): + continue + weight = node.args[1] + observer = weight if weight.op == "call_module" else weight.args[0] + return type(modules[str(observer.target)]).__name__ + raise AssertionError(f"no conv under {conv_name_fragment}") + + +def test_module_name_config_reaches_fused_conv_relu(): + model = FeaturesAndClassifier().eval() + example = (torch.randn(1, 3, 8, 8),) + per_tensor = get_symmetric_quantization_config(is_per_channel=False) + quantizer = EspNNQuantizer().set_module_name("classifier", per_tensor) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), quantizer + ) + assert "PerChannel" in _weight_observer(prepared, "features") + assert "PerChannel" not in _weight_observer(prepared, "classifier") + + +def test_hardtanh_bounds_given_as_kwargs_fuse_as_relu6(): + graph = torch.fx.Graph() + x = graph.placeholder("x") + by_kwargs = graph.call_function( + torch.ops.aten.hardtanh.default, (x,), {"min_val": 0.0, "max_val": 6.0} + ) + by_args = graph.call_function(torch.ops.aten.hardtanh.default, (x, 0.0, 6.0)) + other = graph.call_function(torch.ops.aten.hardtanh.default, (x,)) + assert fused_activation_kind(by_kwargs) == "relu6" + assert fused_activation_kind(by_args) == "relu6" + assert fused_activation_kind(other) is None + + +def test_preprocess_rejects_unknown_format_spec(): + with pytest.raises(ValueError, match="espnn_format"): + EspNNBackend.preprocess(None, [CompileSpec(INT8_FORMAT_SPEC.key, b"float")]) + + +def test_runtime_limits_accept_a_lowered_graph(): + check_runtime_limits(_small_graph()) + + +def test_runtime_limits_reject_too_many_outputs(): + graph = _small_graph() + graph = dataclasses.replace(graph, outputs=list(range(MAX_GRAPH_IO + 1))) + with pytest.raises(ValueError, match="outputs"): + check_runtime_limits(graph) + + +def test_runtime_limits_reject_too_many_operations(): + graph = _small_graph() + operations = graph.operations * (MAX_GRAPH_OPERATIONS // len(graph.operations) + 1) + with pytest.raises(ValueError, match="operations"): + check_runtime_limits(dataclasses.replace(graph, operations=operations)) + + +def test_runtime_limits_reject_v6_opcodes_in_v5_graphs(): + graph = _small_graph() + copy = dataclasses.replace(graph.operations[0], opcode=XTS3OpCode.COPY) + with pytest.raises(ValueError, match="version 6"): + check_runtime_limits( + dataclasses.replace( + graph, version=5, operations=[copy], inputs=[], outputs=[] + ) + ) + + +def test_runtime_limits_reject_misaligned_int32_constants(): + graph = _small_graph() + tensors = [ + ( + dataclasses.replace(tensor, data_offset=tensor.data_offset + 2) + if tensor.flags & TENSOR_FLAG_CONSTANT and tensor.dtype == XTS3DType.INT32 + else tensor + ) + for tensor in graph.tensors + ] + with pytest.raises(ValueError, match="4-byte aligned"): + check_runtime_limits(dataclasses.replace(graph, tensors=tensors)) + + +class DropoutClassifier(nn.Module): + """MobileNetV3's classifier: Linear, Hardswish, in-place Dropout, Linear.""" + + def __init__(self): + super().__init__() + self.head = nn.Sequential( + nn.Linear(16, 32), + nn.Hardswish(inplace=True), + nn.Dropout(0.2, inplace=True), + nn.Linear(32, 10), + ) + + def forward(self, x): + return self.head(x) + + +def test_dropout_between_quantized_ops_reads_the_producer(): + converted, program = quantize(DropoutClassifier(), [(1, 16)]) + graph = build_graph(program) + check_dataflow(graph) + last = graph.operations[-1] + assert last.opcode == XTS3OpCode.LINEAR + assert last.inputs[0] in graph.operations[-2].outputs + + edge, report = to_espnn_edge(program, allow_cpu_fallback=False) + assert report["partitions"] == 1 + x = torch.randn(1, 16) + expected = converted(x) + actual = ProgramEmulator(edge.exported_program())(x).to(torch.float32) + assert torch.allclose(actual, expected, atol=0.2) + + +def test_dataflow_rejects_reading_an_unwritten_activation(): + graph = _small_graph() + unwritten = dataclasses.replace( + graph.tensors[graph.operations[-1].inputs[0]], + tensor_id=len(graph.tensors), + ) + last = dataclasses.replace( + graph.operations[-1], + inputs=[unwritten.tensor_id, *graph.operations[-1].inputs[1:]], + ) + broken = dataclasses.replace( + graph, + tensors=[*graph.tensors, unwritten], + operations=[*graph.operations[:-1], last], + ) + with pytest.raises(ValueError, match="nothing writes"): + check_dataflow(broken) diff --git a/backends/espressif/test/test_cpu_fallback.py b/backends/espressif/test/test_cpu_fallback.py new file mode 100644 index 00000000000..a43d186cb7c --- /dev/null +++ b/backends/espressif/test/test_cpu_fallback.py @@ -0,0 +1,171 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for partial delegation: int8 regions run on the ESP-NN delegate, +unsupported operators stay on the ExecuTorch CPU (portable) kernels. + + pytest backends/espressif/test/test_cpu_fallback.py +""" + +from __future__ import annotations + +import math + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.lowering import to_espnn_edge + +from executorch.backends.espressif.reference.program_emulator import ProgramEmulator + +from executorch.backends.espressif.test.test_int8_ops import quantize +from executorch.exir.lowered_backend_module import LoweredBackendModule + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def sqnr_db(reference: torch.Tensor, actual: torch.Tensor) -> float: + reference = reference.to(torch.float64) + noise = (reference - actual.to(torch.float64)).pow(2).sum().item() + signal = reference.pow(2).sum().item() + if noise == 0: + return math.inf + return 10 * math.log10(signal / noise) + + +class CumsumHead(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + self.out = nn.Conv2d(8, 4, 1) + + def forward(self, x): + y = F.relu(self.conv(x)) + y = torch.cumsum(y, dim=3) + return self.out(y) + + +class TinyTransformer(nn.Module): + def __init__(self, dim=16, heads=2, classes=5): + super().__init__() + self.heads = heads + self.stem = nn.Conv2d(3, dim, 4, stride=4) + self.norm1 = nn.LayerNorm(dim) + self.qkv = nn.Linear(dim, 3 * dim) + self.proj = nn.Linear(dim, dim) + self.norm2 = nn.LayerNorm(dim) + self.mlp1 = nn.Linear(dim, 2 * dim) + self.mlp2 = nn.Linear(2 * dim, dim) + self.head = nn.Linear(dim, classes) + + def forward(self, x): + tokens = self.stem(x).flatten(2).transpose(1, 2) + n, t, d = tokens.shape + h = self.norm1(tokens) + q, k, v = self.qkv(h).chunk(3, dim=-1) + q = q.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + k = k.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + v = v.reshape(n, t, self.heads, d // self.heads).transpose(1, 2) + attention = torch.softmax( + q @ k.transpose(-2, -1) / math.sqrt(d // self.heads), dim=-1 + ) + mixed = (attention @ v).transpose(1, 2).reshape(n, t, d) + tokens = tokens + self.proj(mixed) + tokens = tokens + self.mlp2(F.gelu(self.mlp1(self.norm2(tokens)))) + return self.head(tokens.mean(dim=1)) + + +class KeywordLSTM(nn.Module): + def __init__(self): + super().__init__() + self.frontend = nn.Linear(10, 16) + self.lstm = nn.LSTM(16, 24, batch_first=True) + self.classifier = nn.Linear(24, 6) + + def forward(self, features): + x = F.relu(self.frontend(features)) + sequence, _ = self.lstm(x) + return self.classifier(sequence[:, -1, :]) + + +class CpuIsland(nn.Module): + def __init__(self): + super().__init__() + self.a = nn.Conv2d(3, 8, 3, padding=1) + self.b = nn.Conv2d(8, 8, 3, padding=1) + self.c = nn.Conv2d(8, 4, 1) + + def forward(self, x): + y = F.relu(self.a(x)) + z = torch.nn.functional.normalize(y, dim=1) + z = F.relu(self.b(z)) + y + return self.c(z) + + +torch.manual_seed(31) +CASES = [ + ("cumsum-on-cpu", CumsumHead(), [(1, 3, 8, 8)], 25.0), + ("tiny-transformer", TinyTransformer(), [(1, 3, 16, 16)], 15.0), + ("lstm-keywords", KeywordLSTM(), [(1, 12, 10)], 15.0), + ("cpu-island-between-delegates", CpuIsland(), [(1, 3, 8, 8)], 20.0), +] + + +def lower(model, shapes): + converted, program = quantize(model, shapes) + edge, report = to_espnn_edge(program) + return converted, edge, report + + +@pytest.mark.parametrize( + "name, model, shapes, min_sqnr", CASES, ids=[c[0] for c in CASES] +) +def test_partial_delegation_tracks_pytorch(name, model, shapes, min_sqnr): + converted, edge, report = lower(model, shapes) + program = edge.exported_program() + delegates = [ + m + for _, m in program.graph_module.named_children() + if isinstance(m, LoweredBackendModule) + ] + assert delegates, f"{name}: nothing was delegated" + assert report["cpu_ops"], f"{name}: expected some operators to fall back to the CPU" + emulator = ProgramEmulator(program) + for trial in range(3): + torch.manual_seed(500 + trial) + inputs = [torch.randn(shape) for shape in shapes] + expected = converted(*inputs) + actual = emulator(*inputs) + expected = expected if isinstance(expected, (tuple, list)) else (expected,) + actual = actual if isinstance(actual, (tuple, list)) else (actual,) + for want, got in zip(expected, actual): + assert got.shape == want.shape + assert ( + sqnr_db(want, got) >= min_sqnr + ), f"{name}: SQNR {sqnr_db(want, got):.1f} dB" + assert len(edge.to_executorch().buffer) > 0 + + +def test_fully_supported_model_has_no_cpu_ops(): + from executorch.backends.espressif.test.test_int8_lowering import SEBlockModel + + _, edge, report = lower(SEBlockModel(), [(1, 3, 16, 16)]) + assert report["partitions"] == 1 + assert report["cpu_ops"] == [] + + +def test_strict_mode_still_rejects_unsupported_operators(): + _, program = quantize(CumsumHead(), [(1, 3, 8, 8)]) + with pytest.raises(Exception, match="cumsum"): + to_espnn_edge(program, allow_cpu_fallback=False) diff --git a/backends/espressif/test/test_int8_lowering.py b/backends/espressif/test/test_int8_lowering.py new file mode 100644 index 00000000000..c35a17f7010 --- /dev/null +++ b/backends/espressif/test/test_int8_lowering.py @@ -0,0 +1,342 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for the ESP32 int8 quantize-and-lower pipeline. + + pytest backends/espressif/test/test_int8_lowering.py +""" + +from __future__ import annotations + +import random +import struct + +import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization.xts3 import ( + _quantize_multiplier, + build_graph, + parse_graph, + serialize_graph, + XTS3_VERSION, + XTS3OpCode, +) +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +INT32_MIN = -(1 << 31) +INT32_MAX = (1 << 31) - 1 + + +def c_srdhm(a: int, b: int) -> int: + if a == b == INT32_MIN: + return INT32_MAX + nudge = (1 << 30) if (a >= 0) == (b >= 0) else 1 - (1 << 30) + product = a * b + nudge + quotient = abs(product) >> 31 + return quotient if product >= 0 else -quotient + + +def c_rounding_divide(x: int, exponent: int) -> int: + mask = (1 << exponent) - 1 + remainder = x & mask + result = x >> exponent + threshold = (mask >> 1) + (1 if result < 0 else 0) + return result + (1 if remainder > threshold else 0) + + +def c_requantize(x: int, multiplier: int, shift: int) -> int: + left = max(shift, 0) + right = max(-shift, 0) + shifted = ((x << left) + (1 << 31)) % (1 << 32) - (1 << 31) + return c_rounding_divide(c_srdhm(shifted, multiplier), right) + + +def test_requantization_matches_c_semantics(): + rng = random.Random(0) + xs, mults, shifts = [], [], [] + for _ in range(20000): + xs.append(rng.randint(-(1 << 24), 1 << 24)) + mults.append(rng.randint(1 << 30, INT32_MAX)) + shifts.append(rng.randint(-20, 2)) + xs += [INT32_MIN, INT32_MAX, 0, -1, 1] + mults += [INT32_MAX, 1 << 30, 1 << 30, INT32_MAX, INT32_MAX] + shifts += [0, -1, -31, -5, 0] + expected = [c_requantize(x, m, s) for x, m, s in zip(xs, mults, shifts)] + actual = reference.multiply_by_quantized_multiplier( + np.array(xs, dtype=np.int64), np.array(mults), np.array(shifts) + ) + assert actual.tolist() == expected + + +def test_srdhm_overflow_case(): + assert ( + int(reference.saturating_rounding_doubling_high_mul(INT32_MIN, INT32_MIN)) + == INT32_MAX + ) + + +@pytest.mark.parametrize("value", [1e-6, 0.0123, 0.5, 0.999999, 1.0, 3.75, 1e3]) +def test_quantize_multiplier_reconstructs_value(value): + multiplier, shift = _quantize_multiplier(value) + assert (1 << 30) <= multiplier <= INT32_MAX + assert abs(multiplier * 2.0 ** (shift - 31) - value) <= value * 2**-30 + + +class ConvActModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.dw = nn.Conv2d(16, 16, 3, padding=1, groups=16) + self.grouped = nn.Conv2d(16, 16, 3, padding=1, groups=4) + self.pw = nn.Conv2d(16, 8, 1) + self.fc = nn.Linear(8 * 2 * 2, 10) + + def forward(self, x): + x = F.relu6(self.conv(x)) + y = F.relu(self.dw(x)) + x = F.relu(self.grouped(y) + x) + x = F.max_pool2d(x, 3, 2, 1) + x = self.pw(x) + x = F.avg_pool2d(F.relu(x), 2) + return self.fc(torch.flatten(x, 1)) + + +class ConcatModel(nn.Module): + def __init__(self): + super().__init__() + self.a = nn.Conv2d(4, 6, 1) + self.b = nn.Conv2d(4, 10, 3, padding=1) + self.head = nn.Linear(16, 5) + + def forward(self, x): + x = torch.cat([F.relu(self.a(x)), F.relu(self.b(x))], dim=1) + x = F.adaptive_avg_pool2d(x, 1) + return self.head(torch.flatten(x, 1)) + + +class SEBlockModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.act = nn.Hardswish() + self.fc1 = nn.Conv2d(16, 8, 1) + self.fc2 = nn.Conv2d(8, 16, 1) + self.gate = nn.Hardsigmoid() + self.dw = nn.Conv2d(16, 16, 3, padding=1, groups=16) + self.head = nn.Linear(16, 6) + + def forward(self, x): + x = self.act(self.conv(x)) + scale = self.gate(self.fc2(F.relu(self.fc1(F.adaptive_avg_pool2d(x, 1))))) + x = x * scale + x = F.silu(self.dw(x)) + x = torch.sigmoid(x) * x + x * 0.5 + 1.0 + x = torch.flatten(F.adaptive_avg_pool2d(x, 1), 1) + return torch.softmax(self.head(x), dim=1) + + +class UnsupportedModel(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 4, 3) + + def forward(self, x): + return torch.cumsum(self.conv(x), dim=1) + + +def quantize(model: nn.Module, shape, samples: int = 16, seed: int = 0): + torch.manual_seed(seed) + model = model.eval() + example = (torch.randn(shape),) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(samples): + prepared(torch.randn(shape)) + converted = convert_pt2e(prepared) + return converted, torch.export.export(converted, example, strict=True) + + +def run_device_model(graph, x: torch.Tensor): + interpreter = reference.ReferenceInterpreter(graph) + source = graph.tensors[interpreter.inputs[0]] + result = graph.tensors[interpreter.outputs[0]] + quantized = reference.to_device_layout( + source, reference.quantize(source, x.numpy()) + ) + output = interpreter.run({source.tensor_id: quantized})[result.tensor_id] + return result, output + + +@pytest.mark.parametrize( + "model, shape", + [ + (ConvActModel(), (1, 3, 16, 16)), + (ConcatModel(), (1, 4, 6, 6)), + (SEBlockModel(), (1, 3, 16, 16)), + ], + ids=["conv-act-pool-linear", "concat-mean-linear", "se-lut-mul-softmax"], +) +def test_lowering_tracks_pytorch_int8(model, shape): + converted, program = quantize(model, shape) + graph = parse_graph(serialize_graph(build_graph(program))) + assert graph.version == XTS3_VERSION + for seed in range(4): + torch.manual_seed(100 + seed) + x = torch.randn(shape) + result, output = run_device_model(graph, x) + expected = converted(x).reshape(-1).numpy() + expected_q = np.clip( + np.rint(expected / result.quant_scale) + result.quant_zero_point, -128, 127 + ) + assert np.max(np.abs(expected_q - output.reshape(-1).astype(np.int64))) <= 2 + + +def test_relu6_is_fused_into_convolution(): + _, program = quantize(ConvActModel(), (1, 3, 16, 16)) + graph = build_graph(program) + opcodes = [op.opcode for op in graph.operations] + assert XTS3OpCode.RELU6 not in opcodes + conv = graph.operations[0] + output = graph.tensors[conv.outputs[0]] + act_min, act_max = struct.unpack_from("= 5 + assert counts.get(XTS3OpCode.SOFTMAX) == 1 + modes = [ + struct.unpack("<5i", op.params)[4] + for op in graph.operations + if op.opcode == XTS3OpCode.MUL + ] + assert 1 in modes and 0 in modes + softmax = [op for op in graph.operations if op.opcode == XTS3OpCode.SOFTMAX][0] + output = graph.tensors[softmax.outputs[0]] + assert output.quant_zero_point == -128 and abs(output.quant_scale - 1 / 256) < 1e-12 + + +def test_lut_matches_pytorch_quantized_function(): + from executorch.backends.espressif.serialization.xts3 import ( + build_lut, + XTS3DType, + XTS3Tensor, + ) + + source = XTS3Tensor( + 0, (1, 4), XTS3DType.INT8, quant_scale=0.05, quant_zero_point=-3 + ) + result = XTS3Tensor( + 1, (1, 4), XTS3DType.INT8, quant_scale=0.03, quant_zero_point=-100 + ) + table = np.frombuffer(build_lut(F.hardswish, source, result), dtype=np.int8) + codes = torch.arange(-128, 128).to(torch.int8) + expected = torch.ops.quantized_decomposed.quantize_per_tensor( + F.hardswish( + torch.ops.quantized_decomposed.dequantize_per_tensor( + codes, 0.05, -3, -128, 127, torch.int8 + ) + ), + 0.03, + -100, + -128, + 127, + torch.int8, + ) + assert table.tolist() == expected.tolist() + + +def test_float_model_cannot_be_lowered_to_int8(): + program = torch.export.export( + ConcatModel().eval(), (torch.randn(1, 4, 6, 6),), strict=True + ) + with pytest.raises(ValueError): + build_graph(program) + + +def test_executorch_delegate_matches_standalone_blob(): + from executorch.backends.espressif.partition.espnn_partitioner import ( + EspNNPartitioner, + ) + from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower + from executorch.exir.lowered_backend_module import LoweredBackendModule + + _, program = quantize(SEBlockModel(), (1, 3, 16, 16)) + standalone = serialize_graph(build_graph(program)) + edge = to_edge_transform_and_lower( + program, + partitioner=[EspNNPartitioner(allow_cpu_fallback=False)], + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + delegates = [ + module + for _, module in edge.exported_program().graph_module.named_children() + if isinstance(module, LoweredBackendModule) + ] + assert len(delegates) == 1 + assert bytes(delegates[0].processed_bytes) == standalone + assert len(edge.to_executorch().buffer) > len(standalone) + + +def test_memory_plan_reuses_buffers(): + _, program = quantize(ConvActModel(), (1, 3, 16, 16)) + graph = build_graph(program) + arena, offsets = reference.plan_activation_arena(graph) + total = sum( + (int(np.prod(t.shape)) + 15) // 16 * 16 + for t in graph.tensors + if not t.is_constant + ) + assert 0 < arena < total + assert all(offset % 16 == 0 for offset in offsets.values()) diff --git a/backends/espressif/test/test_int8_ops.py b/backends/espressif/test/test_int8_ops.py new file mode 100644 index 00000000000..ee33d41fd23 --- /dev/null +++ b/backends/espressif/test/test_int8_ops.py @@ -0,0 +1,322 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Host tests for the XTS3 v6 additions: data movement (COPY), upsampling, +transposed and dilated convolution, broadcasting, padding, any-axis concat, +split, and multiple graph inputs and outputs. + + pytest backends/espressif/test/test_int8_ops.py +""" + +from __future__ import annotations + +import pytest + +np = pytest.importorskip("numpy") +pytest.importorskip("torchao") + +import torch +import torch.nn as nn +import torch.nn.functional as F +from executorch.backends.espressif.quantizer.espnn_quantizer import EspNNQuantizer + +from executorch.backends.espressif.reference import esp_nn_reference as reference +from executorch.backends.espressif.serialization.xts3 import ( + build_graph, + parse_graph, + serialize_graph, + XTS3_VERSION, + XTS3OpCode, +) +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +@pytest.fixture(autouse=True) +def disable_grad(): + with torch.no_grad(): + yield + + +def quantize(model: nn.Module, shapes, samples: int = 24, seed: int = 0): + torch.manual_seed(seed) + model = model.eval() + example = tuple(torch.randn(shape) for shape in shapes) + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), EspNNQuantizer() + ) + for _ in range(samples): + prepared(*(torch.randn(shape) for shape in shapes)) + converted = convert_pt2e(prepared) + return converted, torch.export.export(converted, example, strict=True) + + +def lower(program): + graph = parse_graph(serialize_graph(build_graph(program))) + assert graph.version == XTS3_VERSION + return graph + + +def run_reference(graph, inputs): + interpreter = reference.ReferenceInterpreter(graph) + device_inputs = [] + for tensor_id, value in zip(interpreter.inputs, inputs): + tensor = graph.tensors[tensor_id] + device_inputs.append( + reference.to_device_layout( + tensor, reference.quantize(tensor, value.numpy()) + ) + ) + outputs = interpreter.run_outputs(device_inputs) + return [ + (graph.tensors[tensor_id], output) + for tensor_id, output in zip(interpreter.outputs, outputs) + ] + + +def max_int8_error(converted, graph, shapes, trials: int = 4) -> int: + worst = 0 + for trial in range(trials): + torch.manual_seed(1000 + trial) + inputs = [torch.randn(shape) for shape in shapes] + expected = converted(*inputs) + if isinstance(expected, torch.Tensor): + expected = (expected,) + results = run_reference(graph, inputs) + assert len(results) == len(expected) + for (tensor, device_output), want in zip(results, expected): + logical = reference.to_logical_layout(tensor, device_output).astype( + np.int64 + ) + assert logical.shape == tuple(want.shape) + want_q = np.clip( + np.rint(want.numpy() / tensor.quant_scale) + tensor.quant_zero_point, + -128, + 127, + ) + worst = max(worst, int(np.max(np.abs(want_q - logical)))) + return worst + + +class DetectionHead(nn.Module): + def __init__(self, anchors=3, classes=5): + super().__init__() + self.anchors, self.classes = anchors, classes + self.backbone = nn.Conv2d(3, 16, 3, stride=2, padding=1) + self.p4 = nn.Conv2d(16, 16, 3, stride=2, padding=1) + self.head3 = nn.Conv2d(16, anchors * (classes + 4), 1) + self.head4 = nn.Conv2d(16, anchors * (classes + 4), 1) + + def _flatten(self, x): + n, _, h, w = x.shape + x = x.view(n, self.anchors, self.classes + 4, h, w) + return x.permute(0, 1, 3, 4, 2).reshape(n, -1, self.classes + 4) + + def forward(self, x): + f3 = F.relu(self.backbone(x)) + f4 = F.relu(self.p4(f3)) + boxes = torch.cat( + [self._flatten(self.head3(f3)), self._flatten(self.head4(f4))], dim=1 + ) + return boxes[..., :4], boxes[..., 4:] + + +class SegmentationDecoder(nn.Module): + def __init__(self): + super().__init__() + self.encoder = nn.Conv2d(3, 8, 3, stride=2, padding=1) + self.dilated = nn.Conv2d(8, 8, 3, padding=2, dilation=2) + self.dilated_dw = nn.Conv2d(8, 8, 3, padding=3, dilation=3, groups=8) + self.up = nn.ConvTranspose2d(8, 6, 3, stride=2, padding=1, output_padding=1) + self.skip = nn.Conv2d(3, 6, 1) + + def forward(self, x): + y = F.relu(self.encoder(x)) + y = F.relu(self.dilated(y)) + y = F.relu(self.dilated_dw(y)) + y = self.up(y) + return F.relu(y + self.skip(x)) + + +class UpsampleModel(nn.Module): + def __init__(self, mode: str, align_corners=None, scale=2.0, size=None): + super().__init__() + self.mode, self.align_corners, self.scale, self.size = ( + mode, + align_corners, + scale, + size, + ) + self.conv = nn.Conv2d(3, 6, 3, stride=2, padding=1) + self.out = nn.Conv2d(6, 4, 1) + + def forward(self, x): + y = F.relu(self.conv(x)) + if self.size is not None: + y = F.interpolate( + y, size=self.size, mode=self.mode, align_corners=self.align_corners + ) + else: + y = F.interpolate( + y, + scale_factor=self.scale, + mode=self.mode, + align_corners=self.align_corners, + ) + return self.out(y) + + +class BroadcastPadSplit(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(4, 8, 3, padding=1) + self.row_gate = nn.Conv2d(4, 8, (3, 1), padding=(1, 0)) + self.register_buffer( + "bias_map", torch.linspace(-1, 1, 8 * 6).reshape(1, 8, 1, 6) + ) + self.mix = nn.Conv2d(4, 4, 1) + + def forward(self, x): + y = self.conv(x) + gate = F.adaptive_avg_pool2d(self.row_gate(x), (1, 6)) + y = F.relu(y + gate) + y = y * torch.sigmoid(gate) + y = y + self.bias_map + a, b = torch.split(y, [4, 4], dim=1) + a = F.pad(a, (1, 1, 2, 0)) + b = F.pad(b, (1, 1, 2, 0)) + return self.mix(torch.cat([a, b], dim=2)) + + +class TwoInputTwoOutput(nn.Module): + def __init__(self): + super().__init__() + self.image = nn.Conv2d(3, 8, 3, stride=2, padding=1) + self.extra = nn.Linear(6, 8) + self.classifier = nn.Linear(8, 4) + self.mask = nn.Conv2d(8, 1, 1) + + def forward(self, image, features): + x = F.relu(self.image(image)) + y = F.relu(self.extra(features)) + pooled = torch.flatten(F.adaptive_avg_pool2d(x, 1), 1) + logits = self.classifier(pooled + y) + mask = torch.sigmoid(self.mask(x)) + return mask, logits + + +class TransposeSelect(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 6, 3, padding=1) + self.fc = nn.Linear(6, 5) + + def forward(self, x): + y = F.relu(self.conv(x)) + tokens = y.flatten(2).transpose(1, 2) + first = tokens[:, 0, :] + last = tokens.select(1, -1) + mixed = torch.stack([first, last], dim=1) + return self.fc(mixed).unsqueeze(-1) + + +torch.manual_seed(20261014) + + +class GroupedConvs(nn.Module): + """Grouped (not depthwise) convolutions: plain, strided and dilated.""" + + def __init__(self): + super().__init__() + self.plain = nn.Conv2d(16, 16, 3, padding=1, groups=4) + self.strided = nn.Conv2d(16, 24, 3, stride=2, padding=1, groups=8) + self.dilated = nn.Conv2d(24, 24, 3, padding=2, dilation=2, groups=3) + + def forward(self, x): + x = F.relu(self.plain(x)) + x = F.relu(self.strided(x)) + return self.dilated(x) + + +CASES = [ + ("grouped-conv", GroupedConvs(), [(1, 16, 12, 12)], 2), + ("detection-head-multi-output", DetectionHead(), [(1, 3, 16, 16)], 2), + ("segmentation-dilated-transposed", SegmentationDecoder(), [(1, 3, 16, 16)], 3), + ("upsample-nearest-x2", UpsampleModel("nearest"), [(1, 3, 16, 16)], 2), + ( + "upsample-bilinear-x2", + UpsampleModel("bilinear", align_corners=False), + [(1, 3, 16, 16)], + 2, + ), + ( + "upsample-bilinear-align", + UpsampleModel("bilinear", align_corners=True, size=(13, 11)), + [(1, 3, 16, 16)], + 2, + ), + ("broadcast-pad-split-cat", BroadcastPadSplit(), [(1, 4, 6, 6)], 2), + ("two-inputs-two-outputs", TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)], 2), + ("transpose-select-stack", TransposeSelect(), [(1, 3, 4, 5)], 2), +] + + +@pytest.mark.parametrize( + "name, model, shapes, tolerance", CASES, ids=[c[0] for c in CASES] +) +def test_v6_lowering_tracks_pytorch_int8(name, model, shapes, tolerance): + converted, program = quantize(model, shapes) + graph = lower(program) + assert len(graph.inputs) == len(shapes) + error = max_int8_error(converted, graph, shapes) + assert error <= tolerance, f"{name}: worst int8 difference {error}" + + +def test_detection_head_uses_copy_and_no_rank5_tensors(): + _, program = quantize(DetectionHead(), [(1, 3, 16, 16)]) + graph = lower(program) + assert any(op.opcode == XTS3OpCode.COPY for op in graph.operations) + assert all(len(t.shape) <= 4 for t in graph.tensors) + assert len(graph.outputs) == 2 + + +def test_io_table_round_trips_order(): + _, program = quantize(TwoInputTwoOutput(), [(1, 3, 12, 12), (1, 6)]) + graph = build_graph(program) + parsed = parse_graph(serialize_graph(graph)) + assert parsed.inputs == graph.inputs and parsed.outputs == graph.outputs + assert [parsed.tensors[i].shape for i in parsed.inputs] == [(1, 3, 12, 12), (1, 6)] + assert [parsed.tensors[i].shape for i in parsed.outputs] == [(1, 1, 6, 6), (1, 4)] + + +def test_constants_are_16_byte_aligned(): + _, program = quantize(BroadcastPadSplit(), [(1, 4, 6, 6)]) + blob = serialize_graph(build_graph(program)) + parsed = parse_graph(blob) + constant_base = len(blob) - len(parsed.constant_data) + assert constant_base % 16 == 0 + + +def test_constant_records_match_their_device_layout(): + """Regression: transposed-conv weights were stored (Cout,kh,kw,Cin) but recorded as (Cin,Cout,kh,kw).""" + _, program = quantize(SegmentationDecoder(), [(1, 3, 16, 16)]) + graph = lower(program) + transposed = [ + op for op in graph.operations if op.opcode == XTS3OpCode.TRANSPOSE_CONV2D + ] + assert transposed + for op in transposed: + weight = graph.tensors[op.inputs[1]] + source = graph.tensors[op.inputs[0]] + result = graph.tensors[op.outputs[0]] + assert weight.shape[0] == result.shape[1] and weight.shape[3] == source.shape[1] + for tensor in graph.tensors: + if tensor.is_constant: + assert tensor.data_size == int(np.prod(tensor.shape)) * ( + 4 if tensor.dtype == 4 else 1 + ) diff --git a/backends/espressif/test/test_xts3_parser_host.py b/backends/espressif/test/test_xts3_parser_host.py new file mode 100644 index 00000000000..00077357b9a --- /dev/null +++ b/backends/espressif/test/test_xts3_parser_host.py @@ -0,0 +1,170 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Builds the device-side XTS3 parser (espnn_runtime/src/xts3_graph.cpp, which +has no ESP-IDF dependencies) for the host and checks that it accepts graphs +produced by the serializer and rejects corrupted ones: overflowing COPY +extents and segment counts, oversized UPSAMPLE tables and misaligned int32 +constants. Skipped when no C++ compiler is available. + + pytest backends/espressif/test/test_xts3_parser_host.py +""" + +from __future__ import annotations + +import os +import shutil +import struct +import subprocess +from pathlib import Path + +import pytest + +pytest.importorskip("torchao") + +from executorch.backends.espressif.serialization import xts3 # noqa: E402 +from executorch.backends.espressif.test import test_int8_ops as ops # noqa: E402 + +RUNTIME = Path(__file__).resolve().parents[1] / "espnn_runtime" +DRIVER = Path(__file__).resolve().parent / "host" / "xts3_parse_main.cpp" +INT32_MAX = 2**31 - 1 + + +@pytest.fixture(scope="module") +def parser(tmp_path_factory): + compiler = os.environ.get("CXX") or shutil.which("c++") or shutil.which("g++") + if compiler is None: + pytest.skip("no C++ compiler") + binary = tmp_path_factory.mktemp("xts3") / "xts3_parse" + sources = [str(RUNTIME / "src" / "xts3_graph.cpp"), str(DRIVER)] + base = [compiler, "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror"] + base += ["-I", str(RUNTIME / "include"), *sources, "-o", str(binary)] + sanitized = base[:1] + ["-fsanitize=address,undefined"] + base[1:] + for command in (sanitized, base): + if subprocess.run(command, capture_output=True).returncode == 0: + break + else: + result = subprocess.run(base, capture_output=True, text=True) + pytest.fail(f"host build of xts3_graph.cpp failed:\n{result.stderr}") + + def run(blobs: dict[str, bytes]) -> dict[str, str]: + folder = binary.parent + paths = [] + for name, blob in blobs.items(): + path = folder / f"{name}.xts3" + path.write_bytes(blob) + paths.append(str(path)) + result = subprocess.run( + [str(binary), *paths], + capture_output=True, + text=True, + env={**os.environ, "UBSAN_OPTIONS": "halt_on_error=1"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + verdicts = {} + for line in result.stdout.splitlines(): + if line.startswith("VERDICT "): + path, verdict = line[len("VERDICT ") :].rsplit(" ", 1) + verdicts[Path(path).stem] = verdict + return verdicts + + return run + + +def _lowered(model, shapes) -> xts3.XTS3Graph: + _, program = ops.quantize(model, shapes) + return xts3.build_graph(program) + + +def _layout(graph: xts3.XTS3Graph): + io_table = 0 + if graph.version >= 6: + io_table = xts3.IO_TABLE_HEADER_SIZE + 4 * ( + len(graph.inputs) + len(graph.outputs) + ) + tensors = xts3.HEADER_SIZE + io_table + operations = tensors + xts3.TENSOR_SIZE * len(graph.tensors) + parameters = operations + xts3.OP_SIZE * len(graph.operations) + offsets, cursor = [], parameters + for operation in graph.operations: + offsets.append(cursor) + cursor += len(operation.params) + return tensors, offsets + + +def _first(graph, opcode) -> int: + return next(i for i, op in enumerate(graph.operations) if op.opcode == opcode) + + +def _patch(blob: bytes, offset: int, fmt: str, *values) -> bytes: + data = bytearray(blob) + struct.pack_into(fmt, data, offset, *values) + return bytes(data) + + +def test_parser_accepts_serialized_graphs(parser): + blobs = { + name: xts3.serialize_graph(_lowered(model, shapes)) + for name, model, shapes, _ in ops.CASES + } + assert set(parser(blobs).values()) == {"accept"} + + +def test_parser_rejects_copy_extent_overflow(parser): + graph = _lowered(ops.TransposeSelect(), [(1, 3, 4, 5)]) + blob = xts3.serialize_graph(graph) + _, offsets = _layout(graph) + segment = offsets[_first(graph, xts3.XTS3OpCode.COPY)] + xts3.COPY_HEADER_SIZE + big = [INT32_MAX] * 3 + [1] * 3 + corrupted = _patch(blob, segment + 4, ".ops.txt, the CPU operators to select when building the runtime. + + python export_model.py --calib-dir imagenette2-160/train \\ + --eval-dir imagenette2-160/val --qat-epochs 5 --qat-lr 1e-3 \\ + --output models/mobilenet_v3_small_224.pte + +MobileNetV3-Small needs QAT; plain PTQ loses most of its accuracy. +""" + +from __future__ import annotations + +import argparse +import copy +import random +from pathlib import Path + +import torch +import torch.nn.functional as F +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from executorch.backends.espressif.reference import ProgramEmulator +from executorch.exir import ExecutorchBackendConfig +from executorch.exir.passes import MemoryPlanningPass +from torchao.quantization.pt2e import ( + disable_observer, + move_exported_model_to_eval, + move_exported_model_to_train, +) +from torchao.quantization.pt2e.quantize_pt2e import ( + convert_pt2e, + prepare_pt2e, + prepare_qat_pt2e, +) + +IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".bmp", ".webp"} +# Imagenette folder names and their ImageNet-1k class indices. +IMAGENETTE_TO_IMAGENET = { + "n01440764": 0, + "n02102040": 217, + "n02979186": 482, + "n03000684": 491, + "n03028079": 497, + "n03394916": 566, + "n03417042": 569, + "n03425413": 571, + "n03445777": 574, + "n03888257": 701, +} +MEAN = (0.485, 0.456, 0.406) +STD = (0.229, 0.224, 0.225) + + +def load_images(directory: Path, count: int, size: int, seed: int): + from PIL import Image + from torchvision import transforms + + transform = transforms.Compose( + [ + transforms.Resize(size * 256 // 224), + transforms.CenterCrop(size), + transforms.ToTensor(), + transforms.Normalize(MEAN, STD), + ] + ) + files = sorted( + p for p in directory.rglob("*") if p.suffix.lower() in IMAGE_SUFFIXES + ) + if not files: + raise SystemExit(f"no images found under {directory}") + random.Random(seed).shuffle(files) + images, labels = [], [] + for path in files[:count]: + images.append(transform(Image.open(path).convert("RGB"))) + labels.append(IMAGENETTE_TO_IMAGENET.get(path.parent.name, -1)) + return torch.stack(images), labels + + +def qat_prepare(model, data, args): + batch = min(args.qat_batch, len(data)) + exported = torch.export.export( + copy.deepcopy(model), + (data[:2],), + dynamic_shapes=({0: torch.export.Dim("batch", min=2, max=1024)},), + strict=True, + ).module() + prepared = prepare_qat_pt2e(exported, EspNNQuantizer(is_qat=True)) + parameters = [p for p in prepared.parameters() if p.is_floating_point()] + for parameter in parameters: + parameter.requires_grad_(True) + steps = max(1, round(args.qat_epochs * max(1, len(data) // batch))) + optimizer = torch.optim.SGD(parameters, lr=args.qat_lr, momentum=0.9) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, steps) + generator = torch.Generator().manual_seed(args.seed) + move_exported_model_to_train(prepared) + with torch.no_grad(): + for start in range(0, min(len(data), 16 * batch), batch): + prepared(data[start : start + batch]) + with torch.enable_grad(): + for step in range(steps): + if step == int(steps * 0.6): + prepared.apply(disable_observer) + inputs = data[torch.randint(0, len(data), (batch,), generator=generator)] + if bool(torch.rand(1, generator=generator) < 0.5): + inputs = inputs.flip(3) + with torch.no_grad(): + teacher = F.softmax(model(inputs), dim=1) + loss = F.kl_div( + F.log_softmax(prepared(inputs), dim=1), teacher, reduction="batchmean" + ) + optimizer.zero_grad() + loss.backward() + optimizer.step() + scheduler.step() + if step % max(1, steps // 10) == 0 or step + 1 == steps: + print(f"QAT step {step + 1}/{steps} loss {loss.item():.4f}") + move_exported_model_to_eval(prepared) + for parameter in parameters: + parameter.requires_grad_(False) + return prepared + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", default="mobilenet_v3_small") + parser.add_argument("--size", type=int, default=224) + parser.add_argument("--calib-dir", type=Path, required=True) + parser.add_argument("--calib-samples", type=int, default=1024) + parser.add_argument("--eval-dir", type=Path) + parser.add_argument("--eval-samples", type=int, default=100) + parser.add_argument("--qat-epochs", type=float, default=0.0) + parser.add_argument("--qat-lr", type=float, default=1e-3) + parser.add_argument("--qat-batch", type=int, default=16) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--output", type=Path, default=Path("models/mobilenet_v3_small_224.pte") + ) + args = parser.parse_args() + + from torchvision import models + + model = getattr(models, args.model)(weights="DEFAULT").eval() + calibration, _ = load_images( + args.calib_dir, args.calib_samples, args.size, args.seed + ) + example = (calibration[:1],) + + if args.qat_epochs > 0: + prepared = qat_prepare(model, calibration, args) + else: + prepared = prepare_pt2e( + torch.export.export(model, example, strict=True).module(), + EspNNQuantizer(), + ) + with torch.no_grad(): + for image in calibration: + prepared(image.unsqueeze(0)) + quantized = convert_pt2e(prepared) + + edge, report = to_espnn_edge(torch.export.export(quantized, example, strict=True)) + print( + f"{report['partitions']} ESP-NN partition(s), " + f"{report['delegated_ops']} delegated operators, " + f"{len(report['cpu_ops'])} on the CPU" + ) + + if args.eval_dir is not None: + images, labels = load_images( + args.eval_dir, args.eval_samples, args.size, args.seed + 1 + ) + emulator = ProgramEmulator(edge.exported_program()) + float_correct = device_correct = agree = 0 + with torch.no_grad(): + for image, label in zip(images, labels): + image = image.unsqueeze(0) + float_top1 = int(model(image).argmax()) + device_top1 = int(emulator(image).to(torch.float32).argmax()) + float_correct += int(float_top1 == label) + device_correct += int(device_top1 == label) + agree += int(float_top1 == device_top1) + print( + f"top-1 on {len(labels)} images: float {float_correct}, " + f"int8 program {device_correct}, agreement {agree}" + ) + + program = edge.to_executorch( + ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass(alloc_graph_input=False) + ) + ) + plan = program.executorch_program.execution_plan[0] + operators = sorted( + f"{op.name}.{op.overload}" + for op in plan.operators + if not op.name.startswith("quantized_decomposed::") + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(program.buffer) + ops_path = args.output.with_suffix(".ops.txt") + ops_path.write_text(",".join(operators) + "\n") + print(f"wrote {args.output} ({len(program.buffer)} bytes) and {ops_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/camera_stream/flash.sh b/examples/espressif/camera_stream/flash.sh new file mode 100755 index 00000000000..3fa14895050 --- /dev/null +++ b/examples/espressif/camera_stream/flash.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Build and flash the camera example from an activated ESP-IDF shell. A running +# viewer.py releases the serial port while this script flashes. +set -euo pipefail +PORT=${1:-${ESPPORT:?usage: flash.sh PORT (or set ESPPORT)}} +cd "$(dirname "$0")" +mkdir -p build +idf.py ${ET_BUILD_DIR:+-DET_BUILD_DIR_PATH="$ET_BUILD_DIR"} build +touch build/.viewer_pause +trap 'rm -f build/.viewer_pause' EXIT +sleep 1 +idf.py -p "$PORT" -b 921600 flash diff --git a/examples/espressif/camera_stream/imagenet_classes.txt b/examples/espressif/camera_stream/imagenet_classes.txt new file mode 100644 index 00000000000..8bdc20a0864 --- /dev/null +++ b/examples/espressif/camera_stream/imagenet_classes.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane bird +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot tank suit +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/examples/espressif/camera_stream/main/CMakeLists.txt b/examples/espressif/camera_stream/main/CMakeLists.txt new file mode 100644 index 00000000000..4a8dfd61e06 --- /dev/null +++ b/examples/espressif/camera_stream/main/CMakeLists.txt @@ -0,0 +1,74 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +get_filename_component( + camera_et_root "${CMAKE_CURRENT_LIST_DIR}/../../../.." ABSOLUTE +) + +# The ExecuTorch platform layer is shared with the executor runner example. +# WHOLE_ARCHIVE keeps it over the runtime's weak defaults. +idf_component_register( + SRCS + "main.cpp" + "camera_stream.cpp" + "et_model.cpp" + "model_data.S" + "${camera_et_root}/examples/espressif/executor_runner/esp_pal.cpp" + INCLUDE_DIRS + "." + REQUIRES + espnn_runtime + esp32-camera + esp_jpeg + esp_timer + esp_rom + esp_driver_usb_serial_jtag + WHOLE_ARCHIVE +) + +set(ET_BUILD_DIR_PATH + "${camera_et_root}/cmake-out-esp" + CACHE PATH "ExecuTorch cross-compiled for this target and installed" +) +# GLOBAL so that the project-level link step and espnn_runtime see the imported +# targets. +set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL TRUE) +find_package( + executorch REQUIRED HINTS "${ET_BUILD_DIR_PATH}/lib/cmake/ExecuTorch" +) +target_include_directories( + ${COMPONENT_LIB} PRIVATE "${camera_et_root}/.." + "${camera_et_root}/runtime/core/portable_type/c10" +) +target_compile_definitions( + ${COMPONENT_LIB} PRIVATE C10_USING_CUSTOM_GENERATED_MACROS +) +set(camera_et_libs executorch executorch_selected_kernels) +if(TARGET quantized_ops_lib) + list(APPEND camera_et_libs quantized_ops_lib) +endif() +target_link_libraries(${COMPONENT_LIB} PRIVATE ${camera_et_libs}) + +idf_build_get_property(project_dir PROJECT_DIR) +if(IS_ABSOLUTE "${CONFIG_CAMERA_STREAM_MODEL}") + set(model "${CONFIG_CAMERA_STREAM_MODEL}") +else() + set(model "${project_dir}/${CONFIG_CAMERA_STREAM_MODEL}") +endif() +if(NOT EXISTS "${model}") + message( + FATAL_ERROR + "Camera model ${model} not found. Export one with " + "examples/espressif/camera_stream/export_model.py and set " + "CONFIG_CAMERA_STREAM_MODEL." + ) +endif() +set_source_files_properties( + "model_data.S" PROPERTIES COMPILE_DEFINITIONS "CAMERA_MODEL_FILE=\"${model}\"" + OBJECT_DEPENDS "${model}" +) diff --git a/examples/espressif/camera_stream/main/Kconfig.projbuild b/examples/espressif/camera_stream/main/Kconfig.projbuild new file mode 100644 index 00000000000..6a5cf38928f --- /dev/null +++ b/examples/espressif/camera_stream/main/Kconfig.projbuild @@ -0,0 +1,74 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +menu "Camera inference stream" + + config CAMERA_STREAM_MODEL + string "ExecuTorch program (.pte) to embed; absolute or relative to this project" + default "models/mobilenet_v3_small_224.pte" + + + choice CAMERA_STREAM_PINOUT + prompt "Camera module pinout" + default CAMERA_STREAM_PINOUT_S3_EYE + + config CAMERA_STREAM_PINOUT_S3_EYE + bool "ESP32-S3-EYE, Freenove ESP32-S3-WROOM CAM, ESP32-S3-CAM N16R8" + + config CAMERA_STREAM_PINOUT_XIAO_SENSE + bool "Seeed XIAO ESP32S3 Sense" + endchoice + + config CAMERA_STREAM_SENSOR_JPEG + bool "Capture JPEG from the sensor (decode on the ESP32 for inference)" + default y + help + JPEG frames are about 15 times smaller than RGB565, so the camera + DMA does not compete with inference for PSRAM bandwidth and the + preview needs no software encoder. Turn off to capture RGB565. + + config CAMERA_STREAM_SENSOR_JPEG_QUALITY + int "Sensor JPEG quality (0 to 63, lower is better)" + range 4 63 + default 12 + depends on CAMERA_STREAM_SENSOR_JPEG + + config CAMERA_STREAM_XCLK_MHZ + int "Camera XCLK frequency in MHz" + range 6 24 + default 10 + help + At 20 MHz an OV3660 on the ESP32-S3 CAM boards drops and corrupts + frames after a while (NO-SOI, frame timeouts, dark frames). + 10 MHz is stable and still gives a 12 fps preview. + + config CAMERA_STREAM_PREVIEW + bool "Stream JPEG preview frames alongside the results" + default y + + config CAMERA_STREAM_PREVIEW_QUALITY + int "Preview JPEG quality (1 to 100, higher is better)" + range 1 100 + default 50 + depends on CAMERA_STREAM_PREVIEW + + config CAMERA_STREAM_PREVIEW_MAX_FPS + int "Preview frame rate limit" + range 1 60 + default 15 + depends on CAMERA_STREAM_PREVIEW + + config CAMERA_STREAM_VFLIP + bool "Flip the image vertically" + default n + + config CAMERA_STREAM_HMIRROR + bool "Mirror the image horizontally" + default n + +endmenu diff --git a/examples/espressif/camera_stream/main/camera_stream.cpp b/examples/espressif/camera_stream/main/camera_stream.cpp new file mode 100644 index 00000000000..4360e16c6ae --- /dev/null +++ b/examples/espressif/camera_stream/main/camera_stream.cpp @@ -0,0 +1,970 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +extern "C" { +extern const uint8_t camera_model_start[]; +extern const uint8_t camera_model_end[]; +} + +namespace executorch { +namespace examples { +namespace espressif { + +namespace { + +constexpr int TOP_K = 5; +constexpr int CROP_NUMERATOR = 224; +constexpr int CROP_DENOMINATOR = 256; +constexpr float IMAGENET_MEAN[3] = {0.485f, 0.456f, 0.406f}; +constexpr float IMAGENET_STD[3] = {0.229f, 0.224f, 0.225f}; +constexpr size_t USB_TX_BUFFER_BYTES = 8 * 1024; +constexpr size_t SENSOR_JPEG_BUFFER_BYTES = 32 * 1024; +constexpr TickType_t USB_WRITE_TIMEOUT = pdMS_TO_TICKS(200); +constexpr size_t RESULT_QUEUE_LENGTH = 4; + +#if CONFIG_CAMERA_STREAM_VFLIP +constexpr int VFLIP = 1; +#else +constexpr int VFLIP = 0; +#endif +#if CONFIG_CAMERA_STREAM_HMIRROR +constexpr int HMIRROR = 1; +#else +constexpr int HMIRROR = 0; +#endif +#if CONFIG_CAMERA_STREAM_PREVIEW +constexpr bool PREVIEW = true; +constexpr int PREVIEW_QUALITY = CONFIG_CAMERA_STREAM_PREVIEW_QUALITY; +constexpr int PREVIEW_MAX_FPS = CONFIG_CAMERA_STREAM_PREVIEW_MAX_FPS; +#else +constexpr bool PREVIEW = false; +constexpr int PREVIEW_QUALITY = 50; +constexpr int PREVIEW_MAX_FPS = 1; +#endif +#if CONFIG_ESPNN_DELEGATE_DUAL_CORE +constexpr bool DUAL_CORE = true; +#else +constexpr bool DUAL_CORE = false; +#endif +#if CONFIG_CAMERA_STREAM_SENSOR_JPEG +constexpr bool SENSOR_JPEG = true; +constexpr int SENSOR_JPEG_QUALITY = CONFIG_CAMERA_STREAM_SENSOR_JPEG_QUALITY; +#else +constexpr bool SENSOR_JPEG = false; +constexpr int SENSOR_JPEG_QUALITY = 12; +#endif + +struct Pinout { + const char* name; + int pwdn, reset, xclk, sda, scl; + int d7, d6, d5, d4, d3, d2, d1, d0; + int vsync, href, pclk; +}; + +#if CONFIG_CAMERA_STREAM_PINOUT_XIAO_SENSE +constexpr Pinout PINOUT = { + "XIAO ESP32S3 Sense", + -1, + -1, + 10, + 40, + 39, + 48, + 11, + 12, + 14, + 16, + 18, + 17, + 15, + 38, + 47, + 13}; +#else +constexpr Pinout PINOUT = { + "ESP32-S3-EYE/Freenove", + -1, + -1, + 15, + 4, + 5, + 16, + 17, + 18, + 12, + 10, + 8, + 9, + 11, + 6, + 7, + 13}; +#endif + +struct Prediction { + int32_t index; + float probability; +}; + +struct Result { + uint32_t seq; + int64_t decode_us; + int64_t preprocess_us; + int64_t invoke_us; + int64_t delegate_us; + int64_t cycle_us; + int64_t latency_us; + uint32_t dropped_results; + uint32_t decode_errors; + Prediction top[TOP_K]; +}; + +struct Shared { + SemaphoreHandle_t state_lock; + QueueHandle_t results; + EtModel* model; + int32_t input_width; + int32_t input_height; + bool decoder_swaps_red_blue; + uint8_t* published_bgr; + bool published_ready; + uint32_t published_seq; +}; + +Shared g_shared{}; + +void* psram_alloc(size_t bytes) { + void* pointer = heap_caps_malloc(bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (pointer == nullptr) + pointer = heap_caps_malloc(bytes, MALLOC_CAP_8BIT); + return pointer; +} + +int64_t frame_time_us(const camera_fb_t* frame) { + return static_cast(frame->timestamp.tv_sec) * 1000000LL + + frame->timestamp.tv_usec; +} + +bool decode_jpeg_rgb888( + const uint8_t* jpeg, + size_t length, + uint8_t* output, + size_t output_size, + int32_t& width, + int32_t& height) { + esp_jpeg_image_cfg_t config{}; + config.indata = const_cast(jpeg); + config.indata_size = static_cast(length); + config.outbuf = output; + config.outbuf_size = static_cast(output_size); + config.out_format = JPEG_IMAGE_FORMAT_RGB888; + config.out_scale = JPEG_IMAGE_SCALE_0; + esp_jpeg_image_output_t image{}; + if (esp_jpeg_decode(&config, &image) != ESP_OK) + return false; + width = image.width; + height = image.height; + return true; +} + +bool probe_decoder_channel_order(bool& swaps_red_blue) { + constexpr int SIDE = 16; + uint8_t bgr[SIDE * SIDE * 3]; + for (int i = 0; i < SIDE * SIDE; ++i) { + bgr[i * 3 + 0] = 0; + bgr[i * 3 + 1] = 0; + bgr[i * 3 + 2] = 255; + } + uint8_t* jpeg = nullptr; + size_t length = 0; + if (!fmt2jpg( + bgr, sizeof(bgr), SIDE, SIDE, PIXFORMAT_RGB888, 90, &jpeg, &length)) + return false; + uint8_t decoded[SIDE * SIDE * 3]; + int32_t width = 0; + int32_t height = 0; + const bool ok = + decode_jpeg_rgb888(jpeg, length, decoded, sizeof(decoded), width, height); + std::free(jpeg); + if (!ok || width != SIDE || height != SIDE) + return false; + const int center = (SIDE / 2 * SIDE + SIDE / 2) * 3; + std::printf( + "Decoder check: pure red decodes to bytes %u,%u,%u\n", + decoded[center], + decoded[center + 1], + decoded[center + 2]); + if (decoded[center] > 200 && decoded[center + 2] < 60) { + swaps_red_blue = false; + return true; + } + if (decoded[center + 2] > 200 && decoded[center] < 60) { + swaps_red_blue = true; + return true; + } + return false; +} + +constexpr char BASE64_ALPHABET[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +size_t base64_encode(const uint8_t* source, size_t length, char* destination) { + size_t o = 0; + size_t i = 0; + for (; i + 2 < length; i += 3) { + const uint32_t v = (static_cast(source[i]) << 16) | + (static_cast(source[i + 1]) << 8) | source[i + 2]; + destination[o++] = BASE64_ALPHABET[(v >> 18) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 12) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 6) & 63]; + destination[o++] = BASE64_ALPHABET[v & 63]; + } + if (i < length) { + uint32_t v = static_cast(source[i]) << 16; + if (i + 1 < length) + v |= static_cast(source[i + 1]) << 8; + destination[o++] = BASE64_ALPHABET[(v >> 18) & 63]; + destination[o++] = BASE64_ALPHABET[(v >> 12) & 63]; + destination[o++] = i + 1 < length ? BASE64_ALPHABET[(v >> 6) & 63] : '='; + destination[o++] = '='; + } + return o; +} + +class UsbLink { + public: + static bool begin() { + usb_serial_jtag_driver_config_t config = + USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(); + config.tx_buffer_size = USB_TX_BUFFER_BYTES; + config.rx_buffer_size = 256; + if (!usb_serial_jtag_is_driver_installed() && + usb_serial_jtag_driver_install(&config) != ESP_OK) + return false; + std::fflush(stdout); + usb_serial_jtag_vfs_use_driver(); + return true; + } + + bool write_line(const char* data, size_t length) { + size_t sent = 0; + while (sent < length) { + const int written = usb_serial_jtag_write_bytes( + data + sent, length - sent, USB_WRITE_TIMEOUT); + if (written <= 0) { + if (sent != 0) + usb_serial_jtag_write_bytes("\n", 1, 0); + host_reading_ = false; + ++failed_writes_; + return false; + } + sent += static_cast(written); + } + host_reading_ = true; + return true; + } + + bool probe() { + if (usb_serial_jtag_write_bytes("\n", 1, USB_WRITE_TIMEOUT) > 0) + host_reading_ = true; + return host_reading_; + } + + bool host_reading() const { + return host_reading_; + } + uint32_t failed_writes() const { + return failed_writes_; + } + + private: + bool host_reading_ = true; + uint32_t failed_writes_ = 0; +}; + +class JpegLine { + public: + bool reserve(size_t bytes) { + if (bytes <= capacity_) + return true; + std::free(buffer_); + buffer_ = static_cast(psram_alloc(bytes)); + capacity_ = buffer_ != nullptr ? bytes : 0; + return buffer_ != nullptr; + } + + size_t build( + const char* kind, + uint32_t seq, + uint32_t result_seq, + int32_t width, + int32_t height, + int64_t encode_us, + const uint8_t* jpeg, + size_t length) { + const size_t needed = 192 + ((length + 2) / 3) * 4; + if (!reserve(needed)) + return 0; + const uint32_t crc = esp_rom_crc32_le(0, jpeg, length); + const int header = std::snprintf( + buffer_, + capacity_, + "#JPG kind=%s seq=%u res=%u w=%ld h=%ld enc_us=%lld len=%u crc=%08lx b64=", + kind, + static_cast(seq), + static_cast(result_seq), + static_cast(width), + static_cast(height), + static_cast(encode_us), + static_cast(length), + static_cast(crc)); + if (header <= 0) + return 0; + size_t total = static_cast(header); + total += base64_encode(jpeg, length, buffer_ + total); + buffer_[total++] = '\n'; + return total; + } + + bool send(UsbLink& link, size_t length) const { + return length != 0 && link.write_line(buffer_, length); + } + + private: + char* buffer_ = nullptr; + size_t capacity_ = 0; +}; + +uint8_t expand5(uint32_t v) { + return static_cast((v << 3) | (v >> 2)); +} +uint8_t expand6(uint32_t v) { + return static_cast((v << 2) | (v >> 4)); +} + +class Resampler { + public: + bool init( + int32_t source_width, + int32_t source_height, + int32_t output_width, + int32_t output_height) { + source_width_ = source_width; + output_width_ = output_width; + output_height_ = output_height; + int32_t side = source_width < source_height ? source_width : source_height; + side = side * CROP_NUMERATOR / CROP_DENOMINATOR; + crop_x_ = (source_width - side) / 2; + crop_y_ = (source_height - side) / 2; + crop_side_ = side; + x_index_ = + static_cast(std::malloc(sizeof(int32_t) * output_width * 2)); + y_index_ = + static_cast(std::malloc(sizeof(int32_t) * output_height * 2)); + x_weight_ = + static_cast(std::malloc(sizeof(uint16_t) * output_width)); + y_weight_ = + static_cast(std::malloc(sizeof(uint16_t) * output_height)); + if (x_index_ == nullptr || y_index_ == nullptr || x_weight_ == nullptr || + y_weight_ == nullptr) + return false; + build(output_width, crop_x_, x_index_, x_weight_); + build(output_height, crop_y_, y_index_, y_weight_); + return true; + } + + void run_rgb565_be(const uint8_t* source, uint8_t* rgb888) const { + for (int32_t oy = 0; oy < output_height_; ++oy) { + const uint8_t* row0 = + source + static_cast(y_index_[oy * 2]) * source_width_ * 2; + const uint8_t* row1 = source + + static_cast(y_index_[oy * 2 + 1]) * source_width_ * 2; + const uint32_t wy = y_weight_[oy]; + uint8_t* out = rgb888 + static_cast(oy) * output_width_ * 3; + for (int32_t ox = 0; ox < output_width_; ++ox) { + const int32_t x0 = x_index_[ox * 2] * 2; + const int32_t x1 = x_index_[ox * 2 + 1] * 2; + const uint32_t wx = x_weight_[ox]; + const uint32_t p00 = + (static_cast(row0[x0]) << 8) | row0[x0 + 1]; + const uint32_t p01 = + (static_cast(row0[x1]) << 8) | row0[x1 + 1]; + const uint32_t p10 = + (static_cast(row1[x0]) << 8) | row1[x0 + 1]; + const uint32_t p11 = + (static_cast(row1[x1]) << 8) | row1[x1 + 1]; + out[ox * 3 + 0] = blend( + expand5(p00 >> 11), + expand5(p01 >> 11), + expand5(p10 >> 11), + expand5(p11 >> 11), + wx, + wy); + out[ox * 3 + 1] = blend( + expand6((p00 >> 5) & 63), + expand6((p01 >> 5) & 63), + expand6((p10 >> 5) & 63), + expand6((p11 >> 5) & 63), + wx, + wy); + out[ox * 3 + 2] = blend( + expand5(p00 & 31), + expand5(p01 & 31), + expand5(p10 & 31), + expand5(p11 & 31), + wx, + wy); + } + } + } + + void run_rgb888(const uint8_t* source, bool swap_red_blue, uint8_t* rgb888) + const { + const int red = swap_red_blue ? 2 : 0; + const int blue = swap_red_blue ? 0 : 2; + for (int32_t oy = 0; oy < output_height_; ++oy) { + const uint8_t* row0 = + source + static_cast(y_index_[oy * 2]) * source_width_ * 3; + const uint8_t* row1 = source + + static_cast(y_index_[oy * 2 + 1]) * source_width_ * 3; + const uint32_t wy = y_weight_[oy]; + uint8_t* out = rgb888 + static_cast(oy) * output_width_ * 3; + for (int32_t ox = 0; ox < output_width_; ++ox) { + const int32_t x0 = x_index_[ox * 2] * 3; + const int32_t x1 = x_index_[ox * 2 + 1] * 3; + const uint32_t wx = x_weight_[ox]; + out[ox * 3 + 0] = blend( + row0[x0 + red], + row0[x1 + red], + row1[x0 + red], + row1[x1 + red], + wx, + wy); + out[ox * 3 + 1] = blend( + row0[x0 + 1], row0[x1 + 1], row1[x0 + 1], row1[x1 + 1], wx, wy); + out[ox * 3 + 2] = blend( + row0[x0 + blue], + row0[x1 + blue], + row1[x0 + blue], + row1[x1 + blue], + wx, + wy); + } + } + } + + int32_t crop_x() const { + return crop_x_; + } + int32_t crop_y() const { + return crop_y_; + } + int32_t crop_side() const { + return crop_side_; + } + + private: + static uint8_t blend( + uint32_t a, + uint32_t b, + uint32_t c, + uint32_t d, + uint32_t wx, + uint32_t wy) { + const uint32_t top = a * (256 - wx) + b * wx; + const uint32_t bottom = c * (256 - wx) + d * wx; + return static_cast((top * (256 - wy) + bottom * wy + 32768) >> 16); + } + + void build(int32_t output, int32_t offset, int32_t* index, uint16_t* weight) + const { + const int32_t last = crop_side_ - 1; + for (int32_t o = 0; o < output; ++o) { + int32_t position = + static_cast( + (static_cast(2 * o + 1) * crop_side_ * 256) / + (2 * output)) - + 128; + if (position < 0) + position = 0; + int32_t i0 = position >> 8; + if (i0 > last) + i0 = last; + const int32_t i1 = i0 < last ? i0 + 1 : last; + index[o * 2] = offset + i0; + index[o * 2 + 1] = offset + i1; + weight[o] = static_cast(i0 < last ? position & 255 : 0); + } + } + + int32_t source_width_ = 0; + int32_t output_width_ = 0; + int32_t output_height_ = 0; + int32_t crop_x_ = 0; + int32_t crop_y_ = 0; + int32_t crop_side_ = 0; + int32_t* x_index_ = nullptr; + int32_t* y_index_ = nullptr; + uint16_t* x_weight_ = nullptr; + uint16_t* y_weight_ = nullptr; +}; + +bool top_k(const float* logits, float* scratch, size_t count, Prediction* top) { + if (logits == nullptr) + return false; + std::memcpy(scratch, logits, sizeof(float) * count); + float maximum = scratch[0]; + for (size_t i = 1; i < count; ++i) { + if (scratch[i] > maximum) + maximum = scratch[i]; + } + double sum = 0.0; + for (size_t i = 0; i < count; ++i) { + scratch[i] = std::exp(scratch[i] - maximum); + sum += scratch[i]; + } + for (int k = 0; k < TOP_K; ++k) { + top[k].index = -1; + top[k].probability = -1.0f; + } + for (size_t i = 0; i < count; ++i) { + const float p = static_cast(scratch[i] / sum); + if (p <= top[TOP_K - 1].probability) + continue; + int k = TOP_K - 1; + while (k > 0 && p > top[k - 1].probability) { + top[k] = top[k - 1]; + --k; + } + top[k].index = static_cast(i); + top[k].probability = p; + } + return true; +} + +void inference_task(void*) { + EtModel& model = *g_shared.model; + const size_t classes = model.info().classes; + const int32_t width = g_shared.input_width; + const int32_t height = g_shared.input_height; + const size_t rgb_bytes = static_cast(width) * height * 3; + uint8_t* rgb = static_cast(psram_alloc(rgb_bytes)); + float* scratch = static_cast(std::malloc(sizeof(float) * classes)); + uint8_t* decoded = nullptr; + size_t decoded_size = 0; + Resampler resampler; + bool resampler_ready = false; + if (rgb == nullptr || scratch == nullptr) { + std::printf("#ERR inference buffers could not be allocated\n"); + vTaskDelete(nullptr); + return; + } + uint32_t seq = 0; + uint32_t dropped = 0; + uint32_t decode_errors = 0; + int64_t previous = esp_timer_get_time(); + while (true) { + camera_fb_t* frame = esp_camera_fb_get(); + if (frame == nullptr) + continue; + const int64_t started = esp_timer_get_time(); + const int64_t captured = frame_time_us(frame); + const int32_t frame_width = static_cast(frame->width); + const int32_t frame_height = static_cast(frame->height); + if (!resampler_ready) { + resampler_ready = + resampler.init(frame_width, frame_height, width, height); + std::printf( + "#CROP x=%ld y=%ld side=%ld\n", + static_cast(resampler.crop_x()), + static_cast(resampler.crop_y()), + static_cast(resampler.crop_side())); + } + int64_t decode_us = 0; + if (SENSOR_JPEG) { + const size_t needed = static_cast(frame_width) * frame_height * 3; + if (decoded_size < needed) { + std::free(decoded); + decoded = static_cast(psram_alloc(needed)); + decoded_size = decoded != nullptr ? needed : 0; + } + int32_t decoded_width = 0; + int32_t decoded_height = 0; + const bool ok = decoded != nullptr && + decode_jpeg_rgb888( + frame->buf, + frame->len, + decoded, + decoded_size, + decoded_width, + decoded_height) && + decoded_width == frame_width && decoded_height == frame_height; + esp_camera_fb_return(frame); + decode_us = esp_timer_get_time() - started; + if (!ok) { + ++decode_errors; + continue; + } + resampler.run_rgb888(decoded, g_shared.decoder_swaps_red_blue, rgb); + } else { + resampler.run_rgb565_be(frame->buf, rgb); + esp_camera_fb_return(frame); + } + model.set_input_rgb888(rgb, IMAGENET_MEAN, IMAGENET_STD); + const int64_t preprocessed = esp_timer_get_time(); + if (!model.run()) + continue; + Result result{}; + result.seq = ++seq; + result.decode_us = decode_us; + result.preprocess_us = preprocessed - started; + result.invoke_us = model.last_run_us(); + result.delegate_us = model.last_delegate_us(); + top_k(model.logits(), scratch, classes, result.top); + const int64_t now = esp_timer_get_time(); + result.cycle_us = now - previous; + result.latency_us = now - captured; + result.dropped_results = dropped; + result.decode_errors = decode_errors; + previous = now; + if (xQueueSend(g_shared.results, &result, 0) != pdTRUE) + ++dropped; + if (PREVIEW) { + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + uint8_t* destination = g_shared.published_bgr; + for (size_t i = 0; i < rgb_bytes; i += 3) { + destination[i] = rgb[i + 2]; + destination[i + 1] = rgb[i + 1]; + destination[i + 2] = rgb[i]; + } + g_shared.published_ready = true; + g_shared.published_seq = result.seq; + xSemaphoreGive(g_shared.state_lock); + } + vTaskDelay(1); + } +} + +void send_result(UsbLink& link, const Result& result) { + char line[352]; + int n = std::snprintf( + line, + sizeof(line), + "#RES seq=%u dec_us=%lld pre_us=%lld infer_us=%lld delegate_us=%lld cycle_us=%lld " + "latency_us=%lld " + "dropped=%u dec_err=%u usb_fail=%u top=", + static_cast(result.seq), + static_cast(result.decode_us), + static_cast(result.preprocess_us), + static_cast(result.invoke_us), + static_cast(result.delegate_us), + static_cast(result.cycle_us), + static_cast(result.latency_us), + static_cast(result.dropped_results), + static_cast(result.decode_errors), + static_cast(link.failed_writes())); + for (int k = 0; k < TOP_K && n > 0 && n < static_cast(sizeof(line)) - 24; + ++k) { + n += std::snprintf( + line + n, + sizeof(line) - n, + "%s%ld:%.4f", + k ? "," : "", + static_cast(result.top[k].index), + static_cast(result.top[k].probability)); + } + if (n <= 0 || n >= static_cast(sizeof(line)) - 1) + return; + line[n++] = '\n'; + link.write_line(line, static_cast(n)); +} + +void send_model_input( + UsbLink& link, + JpegLine& line, + uint8_t* input_copy, + size_t rgb_bytes) { + bool have_input = false; + uint32_t input_seq = 0; + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + if (g_shared.published_ready) { + std::memcpy(input_copy, g_shared.published_bgr, rgb_bytes); + g_shared.published_ready = false; + input_seq = g_shared.published_seq; + have_input = true; + } + xSemaphoreGive(g_shared.state_lock); + if (!have_input || !link.host_reading()) + return; + uint8_t* jpeg = nullptr; + size_t length = 0; + const int64_t started = esp_timer_get_time(); + if (fmt2jpg( + input_copy, + rgb_bytes, + static_cast(g_shared.input_width), + static_cast(g_shared.input_height), + PIXFORMAT_RGB888, + PREVIEW_QUALITY, + &jpeg, + &length)) { + line.send( + link, + line.build( + "input", + input_seq, + input_seq, + g_shared.input_width, + g_shared.input_height, + esp_timer_get_time() - started, + jpeg, + length)); + } + std::free(jpeg); +} + +void send_preview_frame(UsbLink& link, JpegLine& line, uint32_t seq) { + camera_fb_t* frame = esp_camera_fb_get(); + if (frame == nullptr) + return; + const int32_t frame_width = static_cast(frame->width); + const int32_t frame_height = static_cast(frame->height); + xSemaphoreTake(g_shared.state_lock, portMAX_DELAY); + const uint32_t result_seq = g_shared.published_seq; + xSemaphoreGive(g_shared.state_lock); + size_t length = 0; + if (SENSOR_JPEG) { + length = line.build( + "cam", + seq, + result_seq, + frame_width, + frame_height, + 0, + frame->buf, + frame->len); + esp_camera_fb_return(frame); + } else { + uint8_t* jpeg = nullptr; + size_t jpeg_length = 0; + const int64_t started = esp_timer_get_time(); + const bool encoded = frame2jpg(frame, PREVIEW_QUALITY, &jpeg, &jpeg_length); + const int64_t encode_us = esp_timer_get_time() - started; + esp_camera_fb_return(frame); + if (encoded) { + length = line.build( + "cam", + seq, + result_seq, + frame_width, + frame_height, + encode_us, + jpeg, + jpeg_length); + } + std::free(jpeg); + } + line.send(link, length); +} + +void stream_task(void*) { + UsbLink link; + if (!link.begin()) { + std::printf("#ERR USB-Serial-JTAG driver install failed\n"); + vTaskDelete(nullptr); + return; + } + JpegLine line; + const size_t rgb_bytes = + static_cast(g_shared.input_width) * g_shared.input_height * 3; + uint8_t* input_copy = + PREVIEW ? static_cast(psram_alloc(rgb_bytes)) : nullptr; + if (PREVIEW && (input_copy == nullptr || !line.reserve(64 * 1024))) { + std::printf("#ERR preview buffers could not be allocated\n"); + vTaskDelete(nullptr); + return; + } + const int64_t period_us = 1000000LL / PREVIEW_MAX_FPS; + int64_t next_preview = esp_timer_get_time(); + uint32_t preview_seq = 0; + while (true) { + TickType_t wait = portMAX_DELAY; + if (PREVIEW) { + const int64_t remaining = next_preview - esp_timer_get_time(); + wait = remaining > 0 ? pdMS_TO_TICKS(remaining / 1000) : 0; + } + Result result; + if (xQueueReceive(g_shared.results, &result, wait) == pdTRUE) { + send_result(link, result); + if (PREVIEW) + send_model_input(link, line, input_copy, rgb_bytes); + continue; + } + if (!PREVIEW) + continue; + const int64_t now = esp_timer_get_time(); + next_preview = now - next_preview > period_us ? now + period_us + : next_preview + period_us; + if (!link.host_reading()) { + next_preview = now + 500000; + link.probe(); + continue; + } + send_preview_frame(link, line, ++preview_seq); + } +} + +bool start_camera() { + camera_config_t config{}; + config.pin_pwdn = PINOUT.pwdn; + config.pin_reset = PINOUT.reset; + config.pin_xclk = PINOUT.xclk; + config.pin_sccb_sda = PINOUT.sda; + config.pin_sccb_scl = PINOUT.scl; + config.pin_d7 = PINOUT.d7; + config.pin_d6 = PINOUT.d6; + config.pin_d5 = PINOUT.d5; + config.pin_d4 = PINOUT.d4; + config.pin_d3 = PINOUT.d3; + config.pin_d2 = PINOUT.d2; + config.pin_d1 = PINOUT.d1; + config.pin_d0 = PINOUT.d0; + config.pin_vsync = PINOUT.vsync; + config.pin_href = PINOUT.href; + config.pin_pclk = PINOUT.pclk; + config.xclk_freq_hz = CONFIG_CAMERA_STREAM_XCLK_MHZ * 1000000; + config.ledc_timer = LEDC_TIMER_0; + config.ledc_channel = LEDC_CHANNEL_0; + config.pixel_format = SENSOR_JPEG ? PIXFORMAT_JPEG : PIXFORMAT_RGB565; + config.frame_size = FRAMESIZE_QVGA; + config.jpeg_quality = SENSOR_JPEG_QUALITY; + config.fb_count = SENSOR_JPEG ? 2 : 3; + config.fb_location = SENSOR_JPEG ? CAMERA_FB_IN_DRAM : CAMERA_FB_IN_PSRAM; + config.grab_mode = CAMERA_GRAB_LATEST; + config.sccb_i2c_port = 0; + config.jpeg_buffer_size = SENSOR_JPEG ? SENSOR_JPEG_BUFFER_BYTES : 0; + const esp_err_t error = esp_camera_init(&config); + if (error != ESP_OK) { + std::printf( + "#ERR camera init failed with pinout %s: 0x%x\n", + PINOUT.name, + static_cast(error)); + return false; + } + sensor_t* sensor = esp_camera_sensor_get(); + const camera_sensor_info_t* info = + sensor ? esp_camera_sensor_get_info(&sensor->id) : nullptr; + if (sensor != nullptr) { + sensor->set_vflip(sensor, VFLIP); + sensor->set_hmirror(sensor, HMIRROR); + } + std::printf( + "Camera: %s (PID 0x%04x) on pinout %s, QVGA %s, XCLK %d MHz\n", + info ? info->name : "unknown", + sensor ? static_cast(sensor->id.PID) : 0U, + PINOUT.name, + SENSOR_JPEG ? "JPEG" : "RGB565", + CONFIG_CAMERA_STREAM_XCLK_MHZ); + return true; +} + +} // namespace + +void run_camera_stream() { + std::printf("\n========================================\n"); + std::printf(" CAMERA INFERENCE STREAM\n"); + std::printf("========================================\n"); + if (SENSOR_JPEG && + !probe_decoder_channel_order(g_shared.decoder_swaps_red_blue)) { + std::printf("#ERR JPEG decoder channel order check failed\n"); + return; + } + if (!start_camera()) + return; + + EtModel* model = new (std::nothrow) EtModel(); + const size_t model_size = reinterpret_cast(camera_model_end) - + reinterpret_cast(camera_model_start); + if (model == nullptr || !model->load(camera_model_start, model_size)) { + std::printf("#ERR model load failed\n"); + return; + } + const EtModelInfo& info = model->info(); + g_shared.model = model; + g_shared.input_width = info.input_width; + g_shared.input_height = info.input_height; + g_shared.state_lock = xSemaphoreCreateMutex(); + g_shared.results = xQueueCreate(RESULT_QUEUE_LENGTH, sizeof(Result)); + g_shared.published_bgr = PREVIEW + ? static_cast(psram_alloc( + static_cast(info.input_width) * info.input_height * 3)) + : nullptr; + if (g_shared.state_lock == nullptr || g_shared.results == nullptr || + (PREVIEW && g_shared.published_bgr == nullptr)) { + std::printf("#ERR stream state could not be allocated\n"); + return; + } + std::printf( + "#CAM model_input=%ldx%ld classes=%u runtime=executorch delegates=%u " + "delegate_arena=%u planned=%u dual_core=%d preview=%d preview_fps=%d " + "sensor=%s sensor_quality=%d xclk_mhz=%d\n", + static_cast(info.input_width), + static_cast(info.input_height), + static_cast(info.classes), + static_cast(info.delegates), + static_cast(info.delegate_arena_bytes), + static_cast(info.planned_bytes), + DUAL_CORE ? 1 : 0, + PREVIEW ? 1 : 0, + PREVIEW_MAX_FPS, + SENSOR_JPEG ? "jpeg" : "rgb565", + SENSOR_JPEG_QUALITY, + CONFIG_CAMERA_STREAM_XCLK_MHZ); + std::printf( + "Internal RAM free %u (largest %u), PSRAM free %u\n", + static_cast( + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_largest_free_block( + MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + static_cast(heap_caps_get_free_size(MALLOC_CAP_SPIRAM))); + xTaskCreatePinnedToCore( + stream_task, "cam_stream", 8 * 1024, nullptr, 4, nullptr, 0); + xTaskCreatePinnedToCore( + inference_task, "cam_infer", 16 * 1024, nullptr, 5, nullptr, 1); +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/camera_stream.h b/examples/espressif/camera_stream/main/camera_stream.h new file mode 100644 index 00000000000..2d2c205e297 --- /dev/null +++ b/examples/espressif/camera_stream/main/camera_stream.h @@ -0,0 +1,26 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +namespace executorch { +namespace examples { +namespace espressif { + +/** + * Loads the embedded model, starts the camera and creates the inference + * (core 1) and USB streaming (core 0) tasks, then returns. On a load or + * camera failure it prints the reason and returns without starting tasks. + */ +void run_camera_stream(); + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/et_model.cpp b/examples/espressif/camera_stream/main/et_model.cpp new file mode 100644 index 00000000000..650aad388fd --- /dev/null +++ b/examples/espressif/camera_stream/main/et_model.cpp @@ -0,0 +1,224 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace executorch { +namespace examples { +namespace espressif { + +using ::executorch::aten::ScalarType; +using ::executorch::aten::Tensor; +using ::executorch::extension::BufferDataLoader; +using ::executorch::runtime::Error; +using ::executorch::runtime::EValue; +using ::executorch::runtime::HierarchicalAllocator; +using ::executorch::runtime::MemoryAllocator; +using ::executorch::runtime::MemoryManager; +using ::executorch::runtime::Method; +using ::executorch::runtime::MethodMeta; +using ::executorch::runtime::Program; +using ::executorch::runtime::Result; +using ::executorch::runtime::Span; +using ::executorch::runtime::etensor::TensorImpl; + +namespace { + +constexpr size_t METHOD_POOL_BYTES = 256 * 1024; +constexpr size_t TEMP_POOL_BYTES = 128 * 1024; + +void* psram_alloc(size_t bytes) { + void* pointer = + heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (pointer == nullptr) + pointer = heap_caps_aligned_alloc(16, bytes, MALLOC_CAP_8BIT); + return pointer; +} + +bool float_tensor(Result<::executorch::runtime::TensorInfo> meta, size_t rank) { + return meta.ok() && meta->scalar_type() == ScalarType::Float && + meta->sizes().size() == rank; +} + +} // namespace + +bool EtModel::load(const uint8_t* data, size_t size) { + ::executorch::runtime::runtime_init(); + auto* loader = new (std::nothrow) BufferDataLoader(data, size); + if (loader == nullptr) + return false; + Result program = Program::load(loader); + if (!program.ok()) { + std::printf("#ERR Program::load failed: 0x%x\n", unsigned(program.error())); + return false; + } + auto* owned_program = new (std::nothrow) Program(std::move(*program)); + if (owned_program == nullptr) + return false; + Result meta = owned_program->method_meta("forward"); + if (!meta.ok() || meta->num_inputs() != 1 || meta->num_outputs() != 1 || + !float_tensor(meta->input_tensor_meta(0), 4) || + !float_tensor(meta->output_tensor_meta(0), 2)) { + std::printf( + "#ERR model must map one float [1,3,H,W] image to float [1,classes]\n"); + return false; + } + auto input_meta = meta->input_tensor_meta(0); + auto output_meta = meta->output_tensor_meta(0); + if (input_meta->sizes()[0] != 1 || input_meta->sizes()[1] != 3 || + output_meta->sizes()[0] != 1) { + std::printf("#ERR model input must be [1,3,H,W]\n"); + return false; + } + + auto* method_pool = static_cast(psram_alloc(METHOD_POOL_BYTES)); + auto* temp_pool = static_cast(psram_alloc(TEMP_POOL_BYTES)); + const size_t planned_count = meta->num_memory_planned_buffers(); + auto* planned_spans = new (std::nothrow) Span[planned_count + 1]; + if (method_pool == nullptr || temp_pool == nullptr || + planned_spans == nullptr) { + std::printf("#ERR runtime pools could not be allocated\n"); + return false; + } + size_t planned_bytes = 0; + for (size_t i = 0; i < planned_count; ++i) { + Result bytes = meta->memory_planned_buffer_size(i); + if (!bytes.ok() || *bytes < 0) + return false; + auto* buffer = + static_cast(psram_alloc(static_cast(*bytes))); + if (buffer == nullptr) { + std::printf( + "#ERR planned buffer %u (%lld bytes) could not be allocated\n", + unsigned(i), + static_cast(*bytes)); + return false; + } + planned_spans[i] = {buffer, static_cast(*bytes)}; + planned_bytes += static_cast(*bytes); + } + + auto* method_allocator = + new (std::nothrow) MemoryAllocator(METHOD_POOL_BYTES, method_pool); + auto* temp_allocator = + new (std::nothrow) MemoryAllocator(TEMP_POOL_BYTES, temp_pool); + auto* planned = + new (std::nothrow) HierarchicalAllocator({planned_spans, planned_count}); + if (method_allocator == nullptr || temp_allocator == nullptr || + planned == nullptr) + return false; + auto* memory = new (std::nothrow) + MemoryManager(method_allocator, planned, temp_allocator); + if (memory == nullptr) + return false; + + const auto before = ::executorch::backends::espressif::delegate_stats(); + Result method = owned_program->load_method("forward", memory); + if (!method.ok()) { + std::printf("#ERR load_method failed: 0x%x\n", unsigned(method.error())); + return false; + } + const auto after = ::executorch::backends::espressif::delegate_stats(); + method_ = new (std::nothrow) Method(std::move(*method)); + if (method_ == nullptr) + return false; + + info_.input_height = input_meta->sizes()[2]; + info_.input_width = input_meta->sizes()[3]; + info_.classes = static_cast(output_meta->sizes()[1]); + info_.planned_bytes = planned_bytes; + info_.delegates = after.delegates - before.delegates; + info_.delegate_arena_bytes = after.arena_bytes - before.arena_bytes; + + const size_t pixels = + static_cast(info_.input_width) * info_.input_height; + input_ = static_cast(psram_alloc(sizeof(float) * pixels * 3)); + if (input_ == nullptr) + return false; + sizes_[0] = 1; + sizes_[1] = 3; + sizes_[2] = info_.input_height; + sizes_[3] = info_.input_width; + for (int i = 0; i < 4; ++i) + dim_order_[i] = static_cast(i); + strides_[3] = 1; + strides_[2] = info_.input_width; + strides_[1] = static_cast(pixels); + strides_[0] = static_cast(pixels * 3); + input_impl_ = new (std::nothrow) + TensorImpl(ScalarType::Float, 4, sizes_, input_, dim_order_, strides_); + return input_impl_ != nullptr; +} + +bool EtModel::set_input_rgb888( + const uint8_t* rgb, + const float mean[3], + const float stddev[3]) { + if (input_ == nullptr || rgb == nullptr) + return false; + float lut[3][256]; + for (int c = 0; c < 3; ++c) { + for (int v = 0; v < 256; ++v) + lut[c][v] = (static_cast(v) / 255.0f - mean[c]) / stddev[c]; + } + const size_t pixels = + static_cast(info_.input_width) * info_.input_height; + float* red = input_; + float* green = input_ + pixels; + float* blue = input_ + 2 * pixels; + for (size_t i = 0; i < pixels; ++i) { + red[i] = lut[0][rgb[i * 3 + 0]]; + green[i] = lut[1][rgb[i * 3 + 1]]; + blue[i] = lut[2][rgb[i * 3 + 2]]; + } + return true; +} + +bool EtModel::run() { + if (method_ == nullptr) + return false; + const int64_t started = esp_timer_get_time(); + const auto before = ::executorch::backends::espressif::delegate_stats(); + Error error = method_->set_input(EValue(Tensor(input_impl_)), 0); + if (error == Error::Ok) + error = method_->execute(); + const auto after = ::executorch::backends::espressif::delegate_stats(); + last_run_us_ = esp_timer_get_time() - started; + last_delegate_us_ = after.total_us - before.total_us; + if (error != Error::Ok) { + std::printf("#ERR execute failed: 0x%x\n", unsigned(error)); + return false; + } + const EValue& output = method_->get_output(0); + return output.isTensor() && + output.toTensor().numel() == static_cast(info_.classes); +} + +const float* EtModel::logits() const { + return method_ != nullptr + ? method_->get_output(0).toTensor().const_data_ptr() + : nullptr; +} + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/et_model.h b/examples/espressif/camera_stream/main/et_model.h new file mode 100644 index 00000000000..21aaa1c7bdf --- /dev/null +++ b/examples/espressif/camera_stream/main/et_model.h @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#pragma once + +#include +#include + +#include +#include + +namespace executorch { +namespace examples { +namespace espressif { + +struct EtModelInfo { + int32_t input_width; + int32_t input_height; + size_t classes; + size_t planned_bytes; + uint32_t delegates; + size_t delegate_arena_bytes; +}; + +class EtModel { + public: + bool load(const uint8_t* program, size_t size); + const EtModelInfo& info() const { + return info_; + } + bool set_input_rgb888( + const uint8_t* rgb, + const float mean[3], + const float stddev[3]); + bool run(); + int64_t last_run_us() const { + return last_run_us_; + } + int64_t last_delegate_us() const { + return last_delegate_us_; + } + const float* logits() const; + + private: + using TensorImpl = ::executorch::runtime::etensor::TensorImpl; + + ::executorch::runtime::Method* method_ = nullptr; + float* input_ = nullptr; + TensorImpl* input_impl_ = nullptr; + TensorImpl::SizesType sizes_[4] = {}; + TensorImpl::DimOrderType dim_order_[4] = {}; + TensorImpl::StridesType strides_[4] = {}; + EtModelInfo info_{}; + int64_t last_run_us_ = 0; + int64_t last_delegate_us_ = 0; +}; + +} // namespace espressif +} // namespace examples +} // namespace executorch diff --git a/examples/espressif/camera_stream/main/idf_component.yml b/examples/espressif/camera_stream/main/idf_component.yml new file mode 100644 index 00000000000..18dc3be57f4 --- /dev/null +++ b/examples/espressif/camera_stream/main/idf_component.yml @@ -0,0 +1,12 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +dependencies: + idf: + version: ">=5.1" + espressif/esp32-camera: "^2.1.8" diff --git a/examples/espressif/camera_stream/main/main.cpp b/examples/espressif/camera_stream/main/main.cpp new file mode 100644 index 00000000000..6b4e8769b48 --- /dev/null +++ b/examples/espressif/camera_stream/main/main.cpp @@ -0,0 +1,28 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include + +#include + +namespace { + +void camera_task(void*) { + executorch::examples::espressif::run_camera_stream(); + vTaskDelete(nullptr); +} + +} // namespace + +extern "C" void app_main() { + xTaskCreatePinnedToCore( + camera_task, "camera_setup", 16 * 1024, nullptr, 5, nullptr, 0); +} diff --git a/examples/espressif/camera_stream/main/model_data.S b/examples/espressif/camera_stream/main/model_data.S new file mode 100644 index 00000000000..564dfdf2f8e --- /dev/null +++ b/examples/espressif/camera_stream/main/model_data.S @@ -0,0 +1,20 @@ +/* + * Copyright 2026 Mwaniki Njagi. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + + .section .rodata.camera_model, "a" + .balign 128 + .global camera_model_start +camera_model_start: + .incbin CAMERA_MODEL_FILE + .global camera_model_end +camera_model_end: + .byte 0 + + .section .note.GNU-stack, "", @progbits diff --git a/examples/espressif/camera_stream/partitions.csv b/examples/espressif/camera_stream/partitions.csv new file mode 100644 index 00000000000..195f4193a6a --- /dev/null +++ b/examples/espressif/camera_stream/partitions.csv @@ -0,0 +1,3 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +factory, app, factory, 0x10000, 15M diff --git a/examples/espressif/camera_stream/sdkconfig.defaults b/examples/espressif/camera_stream/sdkconfig.defaults new file mode 100644 index 00000000000..da0678a871c --- /dev/null +++ b/examples/espressif/camera_stream/sdkconfig.defaults @@ -0,0 +1,18 @@ +CONFIG_IDF_TARGET="esp32s3" +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHMODE_QIO=y +CONFIG_ESPTOOLPY_FLASHFREQ_80M=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_SPIRAM=y +CONFIG_SPIRAM_MODE_OCT=y +CONFIG_SPIRAM_SPEED_80M=y +CONFIG_SPIRAM_USE_MALLOC=y +CONFIG_ESP32S3_DATA_CACHE_64KB=y +CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y +CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB=y +CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y +CONFIG_ESP_TASK_WDT_TIMEOUT_S=30 diff --git a/examples/espressif/camera_stream/viewer.py b/examples/espressif/camera_stream/viewer.py new file mode 100644 index 00000000000..1ae3a0b45aa --- /dev/null +++ b/examples/espressif/camera_stream/viewer.py @@ -0,0 +1,797 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Live viewer for the ESP32-S3 camera inference stream (apps/camera_stream). + +The board prints text lines on its USB-Serial-JTAG port: + #CAM ... stream configuration + #CROP x= y= side= centre crop fed to the model, in camera pixels + #RES seq= pre_us= infer_us= delegate_us= cycle_us= top=idx:prob,... + #JPG kind=cam|input seq= res= w= h= enc_us= len= crc= b64=... + +Terminal dashboard (stdlib only; --preview draws the frame with Pillow): + python examples/espressif/camera_stream/viewer.py --port /dev/ttyACM0 [--preview] + +Chrome dashboard on http://127.0.0.1:8765 (stdlib only): + python examples/espressif/camera_stream/viewer.py --web + +Automated check that prints a JSON summary and exits: + python examples/espressif/camera_stream/viewer.py --duration 30 --summary-json out.json +""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import fcntl +import http.server +import io +import json +import os +import re +import select +import shutil +import socketserver +import struct +import sys +import termios +import threading +import time +import tty +import zlib +from pathlib import Path + +TOOLS_DIR = Path(__file__).resolve().parent +TIOCM_DTR = 0x002 +TIOCM_RTS = 0x004 +INJECTED_LOG = re.compile(rb"cam_hal: [^\r\n]{0,96}\r\n") + + +def load_labels(path: Path) -> list[str]: + if path.is_file(): + return [line.strip() for line in path.read_text().splitlines()] + return [] + + +def parse_fields(text: str) -> dict[str, str]: + fields = {} + for item in text.split(): + if "=" in item: + key, value = item.split("=", 1) + fields[key] = value + return fields + + +class SerialPort: + def __init__(self, path: str): + self.fd = os.open(path, os.O_RDWR | os.O_NOCTTY) + tty.setraw(self.fd) + attributes = termios.tcgetattr(self.fd) + attributes[4] = attributes[5] = termios.B115200 + attributes[6][termios.VMIN] = 1 + attributes[6][termios.VTIME] = 0 + termios.tcsetattr(self.fd, termios.TCSANOW, attributes) + + def _set_lines(self, dtr: bool, rts: bool) -> None: + bits = struct.unpack( + "I", fcntl.ioctl(self.fd, termios.TIOCMGET, struct.pack("I", 0)) + )[0] + bits = (bits | TIOCM_DTR) if dtr else (bits & ~TIOCM_DTR) + bits = (bits | TIOCM_RTS) if rts else (bits & ~TIOCM_RTS) + fcntl.ioctl(self.fd, termios.TIOCMSET, struct.pack("I", bits)) + + def reset(self) -> None: + self._set_lines(dtr=False, rts=True) + time.sleep(0.1) + self._set_lines(dtr=False, rts=False) + + def read(self) -> bytes: + ready, _, _ = select.select([self.fd], [], [], 0.5) + if not ready: + return b"\0" + return os.read(self.fd, 65536) + + def close(self) -> None: + try: + os.close(self.fd) + except OSError: + pass + + +class StreamState: + def __init__(self, labels: list[str]): + self.labels = labels + self.lock = threading.Lock() + self.changed = threading.Condition(self.lock) + self.version = 0 + self.config: dict[str, str] = {} + self.crop: dict[str, int] = {} + self.result: dict | None = None + self.frames: dict[str, dict] = {} + self.logs = collections.deque(maxlen=12) + self.errors = collections.deque(maxlen=12) + self.result_times = collections.deque(maxlen=20) + self.frame_times = collections.deque(maxlen=60) + self.byte_times = collections.deque(maxlen=400) + self.infer_history = collections.deque(maxlen=120) + self.counts = collections.Counter() + self.started = time.time() + self.first_result_at: float | None = None + self.camera_log = "" + self.link = "starting" + + def label(self, index: int) -> str: + if 0 <= index < len(self.labels): + return self.labels[index] + return f"class {index}" + + def note_camera_log(self, messages: list[bytes]) -> None: + with self.lock: + for message in messages: + text = message.decode("utf-8", errors="replace").strip() + key = ( + "camera_overruns" + if "FB-OVF" in text or "FB-SIZE" in text + else "camera_logs" + ) + self.counts[key] += 1 + self.camera_log = text + + def add_bytes(self, count: int) -> None: + with self.lock: + self.byte_times.append((time.time(), count)) + + def handle_line(self, raw: bytes) -> None: + line = raw.decode("utf-8", errors="replace").rstrip("\r\n") + if not line: + return + now = time.time() + if line.startswith("#JPG "): + self._handle_jpeg(line, now) + return + with self.lock: + if line.startswith("#RES "): + fields = parse_fields(line[5:]) + top = [] + for item in fields.get("top", "").split(","): + if ":" in item: + index, probability = item.split(":", 1) + top.append( + { + "index": int(index), + "label": self.label(int(index)), + "probability": float(probability), + } + ) + self.result = { + "seq": int(fields.get("seq", 0)), + "pre_ms": int(fields.get("pre_us", 0)) / 1000.0, + "infer_ms": int(fields.get("infer_us", 0)) / 1000.0, + "delegate_ms": int(fields.get("delegate_us", 0)) / 1000.0, + "cycle_ms": int(fields.get("cycle_us", 0)) / 1000.0, + "latency_ms": int(fields.get("latency_us", 0)) / 1000.0, + "decode_ms": int(fields.get("dec_us", 0)) / 1000.0, + "decode_errors": int(fields.get("dec_err", 0)), + "dropped": int(fields.get("dropped", 0)), + "usb_fail": int(fields.get("usb_fail", 0)), + "top": top, + "host_time": now, + } + self.result_times.append(now) + self.infer_history.append(self.result["infer_ms"]) + self.counts["results"] += 1 + if self.first_result_at is None: + self.first_result_at = now + elif line.startswith("#CAM "): + self.config = parse_fields(line[5:]) + elif line.startswith("#CROP "): + self.crop = {k: int(v) for k, v in parse_fields(line[6:]).items()} + elif line.startswith("#ERR"): + self.errors.append(line) + self.counts["device_errors"] += 1 + else: + self.logs.append(line) + self.version += 1 + self.changed.notify_all() + + def _handle_jpeg(self, line: str, now: float) -> None: + head, _, payload = line.partition(" b64=") + fields = parse_fields(head[5:]) + try: + data = base64.b64decode(payload.strip(), validate=True) + valid = len(data) == int(fields["len"]) and zlib.crc32(data) == int( + fields["crc"], 16 + ) + except (ValueError, KeyError): + valid = False + with self.lock: + if not valid: + self.counts["corrupt_frames"] += 1 + return + kind = fields.get("kind", "cam") + self.frames[kind] = { + "jpeg": data, + "seq": int(fields.get("seq", 0)), + "res": int(fields.get("res", 0)), + "width": int(fields.get("w", 0)), + "height": int(fields.get("h", 0)), + "encode_ms": int(fields.get("enc_us", 0)) / 1000.0, + "host_time": now, + } + self.counts[f"{kind}_frames"] += 1 + if kind == "cam": + self.frame_times.append(now) + self.version += 1 + self.changed.notify_all() + + @staticmethod + def _rate(times: collections.deque) -> float: + if len(times) < 2 or times[-1] == times[0]: + return 0.0 + return (len(times) - 1) / (times[-1] - times[0]) + + def stats(self) -> dict: + now = time.time() + window = [(t, n) for t, n in self.byte_times if now - t <= 3.0] + throughput = sum(n for _, n in window) / 3.0 if window else 0.0 + result = self.result + history = list(self.infer_history) + return { + "inference_fps": round(self._rate(self.result_times), 3), + "preview_fps": round(self._rate(self.frame_times), 2), + "link_kbps": round(throughput / 1024.0, 1), + "result_age_ms": ( + round((now - result["host_time"]) * 1000.0) if result else None + ), + "infer_ms_mean": round(sum(history) / len(history), 1) if history else None, + "infer_ms_min": min(history) if history else None, + "infer_ms_max": max(history) if history else None, + "counts": dict(self.counts), + "latency_ms": result["latency_ms"] if result else None, + "last_camera_log": self.camera_log, + "uptime_s": round(now - self.started, 1), + "link": self.link, + "config": self.config, + "crop": self.crop, + } + + def snapshot(self) -> dict: + with self.lock: + frame = self.frames.get("cam") + return { + "stats": self.stats(), + "result": self.result, + "frame": ( + {k: v for k, v in frame.items() if k != "jpeg"} if frame else None + ), + "infer_history": list(self.infer_history), + "errors": list(self.errors), + "logs": list(self.logs), + } + + +def set_link(state: StreamState, text: str) -> None: + with state.lock: + state.link = text + state.logs.append(f"viewer: {text}") + state.version += 1 + state.changed.notify_all() + + +def reader_thread( # noqa: C901 + path: str, reset: bool, pause_file: Path, state: StreamState, raw_log +) -> None: + pending = bytearray() + port = None + while True: + if port is None: + if pause_file.exists(): + if state.link != "paused for flashing": + set_link(state, "paused for flashing") + time.sleep(0.3) + continue + try: + port = SerialPort(path) + except OSError: + if state.link != "waiting for board": + set_link(state, "waiting for board") + time.sleep(0.5) + continue + if reset: + port.reset() + reset = False + pending.clear() + set_link(state, "connected") + if pause_file.exists(): + port.close() + port = None + continue + try: + chunk = port.read() + except OSError: + chunk = b"" + if not chunk: + port.close() + port = None + time.sleep(0.3) + continue + if chunk == b"\0": + continue + if raw_log is not None: + raw_log.write(chunk) + state.add_bytes(len(chunk)) + pending += chunk + if b"cam_hal: " in pending: + injected = INJECTED_LOG.findall(pending) + if injected: + pending = bytearray(INJECTED_LOG.sub(b"", pending)) + state.note_camera_log(injected) + while True: + newline = pending.find(b"\n") + if newline < 0: + break + line = bytes(pending[: newline + 1]) + del pending[: newline + 1] + state.handle_line(line) + if len(pending) > 4 * 1024 * 1024: + pending.clear() + + +PAGE = r""" + + +ESP32-S3 Live Inference + + +

ESP32-S3 live inference

+
connecting
+
+
+

Camera preview

+
camera stream
+
waiting for first result
+
+
+
-
inferences / s
+
-
inference ms
+
-
in ESP-NN ms
+
-
preprocess ms
+
-
preview fps
+
USB KB/s
+
-
capture to result ms
+
+
+
+
+

Top 5

+
+
+
+

What the model sees

+
model input
+
+
+

Inference latency

+ +
+
+

Device log

+

+    
+
+
+ + +""" + + +def make_handler(state: StreamState): # noqa: C901 + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args) -> None: + pass + + def _send(self, code: int, content_type: str, body: bytes) -> None: + self.send_response(code) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + path = self.path.split("?", 1)[0] + try: + if path == "/": + self._send(200, "text/html; charset=utf-8", PAGE.encode()) + elif path == "/state.json": + self._send( + 200, "application/json", json.dumps(self._snapshot()).encode() + ) + elif path in ("/frame.jpg", "/input.jpg"): + kind = "cam" if path == "/frame.jpg" else "input" + with state.lock: + frame = state.frames.get(kind) + if frame is None: + self._send(404, "text/plain", b"no frame yet") + else: + self._send(200, "image/jpeg", frame["jpeg"]) + elif path == "/stream.mjpg": + self._mjpeg() + elif path == "/events": + self._events() + else: + self._send(404, "text/plain", b"not found") + except (BrokenPipeError, ConnectionResetError): + pass + + def _snapshot(self) -> dict: + snapshot = state.snapshot() + with state.lock: + frame = state.frames.get("input") + snapshot["input_seq"] = frame["seq"] if frame else None + return snapshot + + def _mjpeg(self) -> None: + self.send_response(200) + self.send_header( + "Content-Type", "multipart/x-mixed-replace; boundary=frame" + ) + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + last = None + while True: + with state.changed: + state.changed.wait_for( + lambda last=last: state.frames.get("cam") is not None + and state.frames["cam"]["seq"] != last, + timeout=5.0, + ) + frame = state.frames.get("cam") + if frame is None or frame["seq"] == last: + continue + last = frame["seq"] + jpeg = frame["jpeg"] + self.wfile.write( + b"--frame\r\nContent-Type: image/jpeg\r\nContent-Length: " + + str(len(jpeg)).encode() + + b"\r\n\r\n" + + jpeg + + b"\r\n" + ) + self.wfile.flush() + + def _events(self) -> None: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-store") + self.send_header("Connection", "close") + self.end_headers() + while True: + with state.changed: + state.changed.wait(timeout=0.5) + self.wfile.write( + b"data: " + json.dumps(self._snapshot()).encode() + b"\n\n" + ) + self.wfile.flush() + time.sleep(0.2) + + return Handler + + +class ThreadingServer(socketserver.ThreadingMixIn, http.server.HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def terminal_frame(jpeg: bytes, columns: int) -> list[str]: + try: + from PIL import Image + except ImportError: + return ["(install Pillow for --preview)"] + image = Image.open(io.BytesIO(jpeg)).convert("RGB") + rows = max(2, int(columns * image.height / image.width) // 2 * 2) + image = image.resize((columns, rows)) + pixels = image.load() + lines = [] + for y in range(0, rows, 2): + parts = [] + for x in range(columns): + top = pixels[x, y] + bottom = pixels[x, y + 1] + parts.append( + f"\x1b[38;2;{top[0]};{top[1]};{top[2]}m\x1b[48;2;{bottom[0]};{bottom[1]};{bottom[2]}m▀" + ) + lines.append("".join(parts) + "\x1b[0m") + return lines + + +def run_terminal(state: StreamState, args) -> None: + interactive = sys.stdout.isatty() and not args.plain + last_seq = None + deadline = time.time() + args.duration if args.duration else None + if interactive: + sys.stdout.write("\x1b[?25l\x1b[2J") + try: + while deadline is None or time.time() < deadline: + time.sleep(0.25 if interactive else 0.1) + snap = state.snapshot() + result = snap["result"] + stats = snap["stats"] + if not interactive: + if result and result["seq"] != last_seq: + last_seq = result["seq"] + top = ", ".join( + f"{t['label']} {t['probability'] * 100:.1f}%" + for t in result["top"][:3] + ) + print( + f"#{result['seq']:4d} infer {result['infer_ms']:6.1f} ms pre {result['pre_ms']:5.1f} ms " + f"latency {result['latency_ms']:6.1f} ms " + f"{stats['inference_fps']:.2f} inf/s preview {stats['preview_fps']:.1f} fps " + f"{stats['link_kbps']:.0f} KB/s | {top}", + flush=True, + ) + continue + width = shutil.get_terminal_size((100, 40)).columns + out = [ + "\x1b[H", + f"\x1b[1mESP32-S3 live inference\x1b[0m {state.config.get('model_input', '')} " + f"dual_core={state.config.get('dual_core', '?')}\x1b[K", + "\x1b[K", + ] + if args.preview: + with state.lock: + frame = state.frames.get("cam") + if frame: + out += list( + terminal_frame( + frame["jpeg"], min(args.preview_columns, width - 2) + ) + ) + out.append("\x1b[K") + if result: + out.append( + f"result #{result['seq']} inference {result['infer_ms']:.1f} ms preprocess " + f"{result['pre_ms']:.1f} ms cycle {result['cycle_ms']:.0f} ms " + f"capture-to-result {result['latency_ms']:.0f} ms\x1b[K" + ) + bar_width = max(10, min(40, width - 50)) + for t in result["top"]: + filled = int(round(t["probability"] * bar_width)) + out.append( + f" {t['label'][:30]:30s} {'#' * filled}{'.' * (bar_width - filled)} " + f"{t['probability'] * 100:5.1f}%\x1b[K" + ) + else: + out.append( + "waiting for the first result (the board runs its self-test first)\x1b[K" + ) + counts = stats["counts"] + out.append("\x1b[K") + out.append( + f"inference {stats['inference_fps']:.2f}/s preview {stats['preview_fps']:.1f} fps " + f"USB {stats['link_kbps']:.0f} KB/s result age {stats['result_age_ms']} ms\x1b[K" + ) + out.append( + f"frames {counts.get('cam_frames', 0)} corrupt {counts.get('corrupt_frames', 0)} " + f"camera overruns {counts.get('camera_overruns', 0)} " + f"device errors {counts.get('device_errors', 0)}\x1b[K" + ) + for line in snap["errors"][-3:] + snap["logs"][-4:]: + out.append(f"\x1b[2m{line[:width - 1]}\x1b[0m\x1b[K") + out.append("\x1b[J") + sys.stdout.write("\n".join(out)) + sys.stdout.flush() + except KeyboardInterrupt: + pass + finally: + if interactive: + sys.stdout.write("\x1b[0m\x1b[?25h\n") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--port", + default=os.environ.get("ESPPORT"), + required="ESPPORT" not in os.environ, + help="serial port of the board (default: $ESPPORT)", + ) + parser.add_argument( + "--labels", type=Path, default=TOOLS_DIR / "imagenet_classes.txt" + ) + parser.add_argument( + "--reset", action="store_true", help="reset the board after opening the port" + ) + parser.add_argument("--web", action="store_true", help="serve the Chrome dashboard") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--http-port", type=int, default=8765) + parser.add_argument( + "--preview", action="store_true", help="draw the camera frame in the terminal" + ) + parser.add_argument("--preview-columns", type=int, default=64) + parser.add_argument( + "--plain", + action="store_true", + help="one line per result instead of a dashboard", + ) + parser.add_argument( + "--duration", type=float, default=0.0, help="exit after this many seconds" + ) + parser.add_argument( + "--summary-json", type=Path, help="write final statistics here on exit" + ) + parser.add_argument("--raw-log", type=Path, help="save the raw serial stream") + parser.add_argument( + "--pause-file", + type=Path, + default=TOOLS_DIR / "build" / ".viewer_pause", + help="the serial port is released while this file exists (flash.sh creates it)", + ) + args = parser.parse_args() + + state = StreamState(load_labels(args.labels)) + raw_log = open(args.raw_log, "wb") if args.raw_log else None + threading.Thread( + target=reader_thread, + args=(args.port, args.reset, args.pause_file, state, raw_log), + daemon=True, + ).start() + + if args.web: + server = ThreadingServer((args.host, args.http_port), make_handler(state)) + threading.Thread(target=server.serve_forever, daemon=True).start() + print(f"Chrome dashboard: http://{args.host}:{args.http_port}/", flush=True) + + run_terminal(state, args) + + with state.lock: + summary = state.stats() + summary["last_result"] = state.result + if args.summary_json: + args.summary_json.write_text(json.dumps(summary, indent=2)) + if args.duration: + print(json.dumps(summary, indent=2)) + if raw_log: + raw_log.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/executor_runner/CMakeLists.txt b/examples/espressif/executor_runner/CMakeLists.txt index e353c3913b7..2551b1f26a0 100644 --- a/examples/espressif/executor_runner/CMakeLists.txt +++ b/examples/espressif/executor_runner/CMakeLists.txt @@ -147,6 +147,11 @@ if(ESP_PLATFORM) list(APPEND esp_runner_libs xnnpack_backend) endif() + # quantize/dequantize kernels for quantized models with CPU regions + if(TARGET quantized_ops_lib) + list(APPEND esp_runner_libs quantized_ops_lib) + endif() + if(EXECUTORCH_ENABLE_EVENT_TRACER) target_compile_definitions(${COMPONENT_LIB} PUBLIC ET_EVENT_TRACER_ENABLED) list(APPEND esp_runner_libs etdump flatccrt) diff --git a/examples/espressif/export_espnn_test_programs.py b/examples/espressif/export_espnn_test_programs.py new file mode 100644 index 00000000000..b9133dd82c6 --- /dev/null +++ b/examples/espressif/export_espnn_test_programs.py @@ -0,0 +1,155 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Export the ESP-NN backend's test models as BundleIO programs (.bpte) for the +ESP32 executor runner (examples/espressif/project with -DET_ESPNN_BACKEND=ON). + +Each program bundles one input set and the expected outputs from the host +reference (ProgramEmulator), which reproduces the delegate bit-exactly. +Fully delegated programs must therefore match to float precision; programs +with CPU regions get a tolerance for the portable kernels' float rounding. + +Writes, into --output-dir: + .bpte one program per case + manifest.json per-program atol/rtol, delegate and CPU operator counts + operators.txt the CPU operators the programs need, for + EXECUTORCH_SELECT_OPS_LIST + + python examples/espressif/export_espnn_test_programs.py --output-dir espnn_tests +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import torch + +# The test models are loaded from the checkout: a non-editable install ships +# only the test modules that other shipped code imports. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from backends.espressif.test import ( # noqa: E402 + test_cpu_fallback as fallback_cases, + test_int8_lowering as lowering_cases, + test_int8_ops as op_cases, +) + +from executorch.backends.espressif import to_espnn_edge # noqa: E402 +from executorch.backends.espressif.reference import ProgramEmulator # noqa: E402 +from executorch.devtools.bundled_program.config import ( # noqa: E402 + MethodTestCase, + MethodTestSuite, +) +from executorch.devtools.bundled_program.core import BundledProgram # noqa: E402 +from executorch.devtools.bundled_program.serialize import ( # noqa: E402 + serialize_from_bundled_program_to_flatbuffer, +) +from executorch.exir.lowered_backend_module import LoweredBackendModule # noqa: E402 + +EXACT_TOLERANCE = 1e-5 +FALLBACK_TOLERANCE = 0.01 + + +def cases(): + yield "conv-act-pool-linear", lowering_cases.ConvActModel(), [(1, 3, 16, 16)] + yield "concat-mean-linear", lowering_cases.ConcatModel(), [(1, 4, 6, 6)] + yield "se-lut-mul-softmax", lowering_cases.SEBlockModel(), [(1, 3, 16, 16)] + for name, model, shapes, _ in op_cases.CASES: + yield name, model, shapes + for name, model, shapes, _ in fallback_cases.CASES: + yield name, model, shapes + + +def as_list(value): + return list(value) if isinstance(value, (tuple, list)) else [value] + + +def export_case(name, model, shapes, seed): + _, program = op_cases.quantize(model, shapes) + edge, report = to_espnn_edge(program) + exported = edge.exported_program() + generator = torch.Generator().manual_seed(seed) + inputs = [torch.randn(shape, generator=generator) for shape in shapes] + expected = [ + t.to(torch.float32) for t in as_list(ProgramEmulator(exported)(*inputs)) + ] + + executorch_program = edge.to_executorch() + plan = executorch_program.executorch_program.execution_plan[0] + operators = sorted({f"{op.name}.{op.overload}" for op in plan.operators}) + bundle = BundledProgram( + executorch_program, + [MethodTestSuite("forward", [MethodTestCase(inputs, expected)])], + ) + + exact = not report["cpu_ops"] + if exact: + atol = rtol = EXACT_TOLERANCE + else: + peak = max(float(t.abs().max()) for t in expected) + atol, rtol = FALLBACK_TOLERANCE * max(peak, 1.0), FALLBACK_TOLERANCE + delegates = sum( + isinstance(module, LoweredBackendModule) + for _, module in exported.graph_module.named_children() + ) + entry = { + "name": name, + "file": f"{name}.bpte", + "atol": float(f"{atol:.6g}"), + "rtol": rtol, + "exact": exact, + "delegates": delegates, + "delegated_ops": report["delegated_ops"], + "cpu_ops": report["cpu_ops"], + "operators": operators, + } + return serialize_from_bundled_program_to_flatbuffer(bundle), entry + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument( + "--only", help="comma separated case names (default: every case)" + ) + args = parser.parse_args() + selected = set(args.only.split(",")) if args.only else None + torch.set_grad_enabled(False) + args.output_dir.mkdir(parents=True, exist_ok=True) + + manifest, operators = [], set() + for name, model, shapes in cases(): + if selected is not None and name not in selected: + continue + if any(entry["name"] == name for entry in manifest): + raise SystemExit(f"duplicate case name: {name}") + blob, entry = export_case(name, model, shapes, args.seed) + (args.output_dir / entry["file"]).write_bytes(blob) + manifest.append(entry) + operators.update(entry["operators"]) + print( + f"{name:34s} delegates={entry['delegates']} " + f"cpu_ops={len(entry['cpu_ops'])} atol={entry['atol']} " + f"({len(blob)} bytes)" + ) + if selected is not None and len(manifest) != len(selected): + missing = selected - {entry["name"] for entry in manifest} + raise SystemExit(f"unknown case names: {sorted(missing)}") + + (args.output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) + (args.output_dir / "operators.txt").write_text(",".join(sorted(operators)) + "\n") + print(f"wrote {len(manifest)} programs to {args.output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/espressif/project/CMakeLists.txt b/examples/espressif/project/CMakeLists.txt index 13a303ffd72..a43760c33c4 100644 --- a/examples/espressif/project/CMakeLists.txt +++ b/examples/espressif/project/CMakeLists.txt @@ -23,5 +23,13 @@ set(EXECUTORCH_ROOT # Add the executor_runner as an extra component set(EXTRA_COMPONENT_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/../executor_runner") +# Register the ESP-NN delegate (backends/espressif) with the runtime +option(ET_ESPNN_BACKEND "Build in the ESP-NN ExecuTorch backend" OFF) +if(ET_ESPNN_BACKEND) + list(APPEND EXTRA_COMPONENT_DIRS + "${EXECUTORCH_ROOT}/backends/espressif/espnn_runtime" + ) +endif() + include($ENV{IDF_PATH}/tools/cmake/project.cmake) project(executorch_esp_runner) diff --git a/examples/espressif/quantize_and_lower.py b/examples/espressif/quantize_and_lower.py new file mode 100644 index 00000000000..96224b71f50 --- /dev/null +++ b/examples/espressif/quantize_and_lower.py @@ -0,0 +1,54 @@ +# Copyright 2026 Mwaniki Njagi. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# SPDX-License-Identifier: BSD-3-Clause + +""" +Minimal example: quantize a torchvision model to int8 with the ESP-NN +quantizer and lower it to an ExecuTorch program for Espressif chips. + + python examples/espressif/quantize_and_lower.py --output mobilenet_v2_espnn.pte +""" + +from __future__ import annotations + +import argparse + +import torch +import torchvision +from executorch.backends.espressif import EspNNQuantizer, to_espnn_edge +from torchao.quantization.pt2e.quantize_pt2e import convert_pt2e, prepare_pt2e + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="mobilenet_v2_espnn.pte") + args = parser.parse_args() + + model = torchvision.models.mobilenet_v2(weights="DEFAULT").eval() + example_inputs = (torch.randn(1, 3, 224, 224),) + + exported = torch.export.export(model, example_inputs).module() + prepared = prepare_pt2e(exported, EspNNQuantizer()) + with torch.no_grad(): + for _ in range(16): + prepared(torch.randn(1, 3, 224, 224)) + quantized = convert_pt2e(prepared) + + program = torch.export.export(quantized, example_inputs) + edge, report = to_espnn_edge(program) + print( + f"{report['partitions']} EspNN partition(s), {report['delegated_ops']} delegated operators, " + f"{len(report['cpu_ops'])} operators left for the CPU" + ) + + with open(args.output, "wb") as file: + file.write(edge.to_executorch().buffer) + print(f"wrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/examples/espressif/test_emulator.py b/examples/espressif/test_emulator.py index cf0d30b2972..bcdee998c4a 100644 --- a/examples/espressif/test_emulator.py +++ b/examples/espressif/test_emulator.py @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -"""Check the emulator log from the ESP32-S3 add/multiply smoke test.""" +"""Check an ESP32-S3 emulator log from a BundleIO (.bpte) runner build.""" import argparse import re @@ -33,7 +33,7 @@ def main() -> None: if marker not in output: raise SystemExit(f"FAIL: Missing {marker!r}; see {args.log}") print( - f"PASS: ESP32-S3 ran {finished[1]} inferences and matched both PyTorch outputs." + f"PASS: ESP32-S3 ran {finished[1]} inferences and matched the bundled reference outputs." ) diff --git a/pytest.ini b/pytest.ini index efefecf1335..33be7d530ba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -87,6 +87,7 @@ testpaths = # backends backends/apple/coreml/test + backends/espressif/test backends/native/test backends/test/harness/tests backends/test/suite/tests