|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +/* |
| 10 | + * A small wrapper over the ExecuTorch runtime for sketches that just want to |
| 11 | + * run a model. |
| 12 | + * |
| 13 | + * Loading a method by hand means asking the program for its method name, |
| 14 | + * asking that for its metadata, counting its planned buffers, allocating a |
| 15 | + * span array, sizing and allocating each buffer, wrapping them in a |
| 16 | + * HierarchicalAllocator and pairing that with a MemoryAllocator in a |
| 17 | + * MemoryManager. That is around sixty lines before any inference happens, and |
| 18 | + * none of it is a decision the sketch author makes. |
| 19 | + * |
| 20 | + * alignas(16) static uint8_t pool[8 * 1024]; |
| 21 | + * static ETModel model(model_pte, sizeof(model_pte), pool, sizeof(pool)); |
| 22 | + * |
| 23 | + * if (!model.begin()) { Serial.println(model.error()); } |
| 24 | + * model.setInput(0, values, 3); |
| 25 | + * if (!model.run()) { Serial.println(model.error()); } |
| 26 | + * const float* out = model.output(0); |
| 27 | + * |
| 28 | + * error() returns a readable reason rather than a code, because a bare hex |
| 29 | + * status is what makes these failures expensive to diagnose. The full runtime |
| 30 | + * API stays available through program() and method() for anything this does |
| 31 | + * not cover. |
| 32 | + */ |
| 33 | + |
| 34 | +#pragma once |
| 35 | + |
| 36 | +#include <ExecuTorch.h> |
| 37 | +#include <cstdint> |
| 38 | +#include <cstdio> |
| 39 | +#include <optional> |
| 40 | +#include <utility> |
| 41 | + |
| 42 | +class ETModel { |
| 43 | + public: |
| 44 | + // The model buffer and the pool must outlive this object, which on Arduino |
| 45 | + // means both are normally static or global. |
| 46 | + ETModel( |
| 47 | + const uint8_t* model_data, |
| 48 | + size_t model_size, |
| 49 | + uint8_t* pool, |
| 50 | + size_t pool_size) |
| 51 | + : loader_(model_data, model_size), |
| 52 | + allocator_(static_cast<uint32_t>(pool_size), pool) {} |
| 53 | + |
| 54 | + // Loads the program and its first method. Call once, from setup(). |
| 55 | + bool begin() { |
| 56 | + auto program = executorch::runtime::Program::load(&loader_); |
| 57 | + if (!program.ok()) { |
| 58 | + return fail("Program::load", static_cast<int>(program.error())); |
| 59 | + } |
| 60 | + program_.emplace(std::move(program.get())); |
| 61 | + |
| 62 | + auto name = program_->get_method_name(0); |
| 63 | + if (!name.ok()) { |
| 64 | + return fail("get_method_name", static_cast<int>(name.error())); |
| 65 | + } |
| 66 | + method_name_ = *name; |
| 67 | + |
| 68 | + auto meta = program_->method_meta(method_name_); |
| 69 | + if (!meta.ok()) { |
| 70 | + return fail("method_meta", static_cast<int>(meta.error())); |
| 71 | + } |
| 72 | + |
| 73 | + // Planned buffers hold the intermediate tensors the memory planner sized |
| 74 | + // at export time. They come out of the same pool as the method itself. |
| 75 | + const size_t count = meta->num_memory_planned_buffers(); |
| 76 | + auto* spans = static_cast<Span*>(allocator_.allocate(count * sizeof(Span))); |
| 77 | + if (spans == nullptr) { |
| 78 | + return fail("allocating the planned-buffer table", pool_hint()); |
| 79 | + } |
| 80 | + for (size_t i = 0; i < count; i++) { |
| 81 | + auto size = meta->memory_planned_buffer_size(i); |
| 82 | + if (!size.ok()) { |
| 83 | + return fail( |
| 84 | + "memory_planned_buffer_size", static_cast<int>(size.error())); |
| 85 | + } |
| 86 | + auto bytes = static_cast<size_t>(size.get()); |
| 87 | + auto* buffer = static_cast<uint8_t*>(allocator_.allocate(bytes)); |
| 88 | + if (buffer == nullptr) { |
| 89 | + return fail("allocating a planned buffer", pool_hint()); |
| 90 | + } |
| 91 | + spans[i] = Span(buffer, bytes); |
| 92 | + } |
| 93 | + |
| 94 | + planned_.emplace(executorch::runtime::Span<Span>(spans, count)); |
| 95 | + manager_.emplace(&allocator_, &*planned_); |
| 96 | + |
| 97 | + auto method = program_->load_method(method_name_, &*manager_); |
| 98 | + if (!method.ok()) { |
| 99 | + // Much the most common failure, and the pool is nearly always why. |
| 100 | + return fail("load_method", static_cast<int>(method.error())); |
| 101 | + } |
| 102 | + method_.emplace(std::move(method.get())); |
| 103 | + return true; |
| 104 | + } |
| 105 | + |
| 106 | + // Points input `index` at `data` without copying it, so `data` must stay |
| 107 | + // alive across run(). Passing a `const` array in flash works for every model |
| 108 | + // here, but an operator that writes to its input would be writing to |
| 109 | + // read-only memory; copy into a RAM buffer first if that is a risk. Not |
| 110 | + // copying is the right default on a board with this little RAM -- the |
| 111 | + // keyword spotting input alone is 2 KB. |
| 112 | + // |
| 113 | + // The shape and dtype come from the model, so only the element count is |
| 114 | + // checked against it. |
| 115 | + bool setInput(size_t index, const float* data, size_t count) { |
| 116 | + if (!method_) { |
| 117 | + return fail("setInput before begin()", 0); |
| 118 | + } |
| 119 | + if (index >= kMaxInputs) { |
| 120 | + return fail("input index above the built-in limit", kMaxInputs); |
| 121 | + } |
| 122 | + auto meta = program_->method_meta(method_name_); |
| 123 | + auto info = meta->input_tensor_meta(index); |
| 124 | + if (!info.ok()) { |
| 125 | + return fail("input_tensor_meta", static_cast<int>(info.error())); |
| 126 | + } |
| 127 | + size_t expected = 1; |
| 128 | + for (size_t d = 0; d < info->sizes().size(); d++) { |
| 129 | + sizes_[index][d] = info->sizes()[d]; |
| 130 | + dim_order_[index][d] = info->dim_order()[d]; |
| 131 | + expected *= static_cast<size_t>(info->sizes()[d]); |
| 132 | + } |
| 133 | + if (expected != count) { |
| 134 | + return fail( |
| 135 | + "input element count does not match the model", |
| 136 | + static_cast<int>(expected)); |
| 137 | + } |
| 138 | + impls_[index].emplace( |
| 139 | + info->scalar_type(), |
| 140 | + static_cast<ssize_t>(info->sizes().size()), |
| 141 | + sizes_[index], |
| 142 | + const_cast<float*>(data), |
| 143 | + dim_order_[index]); |
| 144 | + tensors_[index].emplace(&*impls_[index]); |
| 145 | + auto status = method_->set_input( |
| 146 | + executorch::runtime::EValue(*tensors_[index]), index); |
| 147 | + if (status != executorch::runtime::Error::Ok) { |
| 148 | + return fail("set_input", static_cast<int>(status)); |
| 149 | + } |
| 150 | + return true; |
| 151 | + } |
| 152 | + |
| 153 | + bool run() { |
| 154 | + if (!method_) { |
| 155 | + return fail("run before begin()", 0); |
| 156 | + } |
| 157 | + auto status = method_->execute(); |
| 158 | + if (status != executorch::runtime::Error::Ok) { |
| 159 | + return fail("execute", static_cast<int>(status)); |
| 160 | + } |
| 161 | + return true; |
| 162 | + } |
| 163 | + |
| 164 | + // Valid until the next run(). |
| 165 | + const float* output(size_t index = 0) { |
| 166 | + const auto* tensor = outputTensor(index); |
| 167 | + return tensor ? tensor->const_data_ptr<float>() : nullptr; |
| 168 | + } |
| 169 | + |
| 170 | + size_t outputCount(size_t index = 0) { |
| 171 | + const auto* tensor = outputTensor(index); |
| 172 | + return tensor ? static_cast<size_t>(tensor->numel()) : 0; |
| 173 | + } |
| 174 | + |
| 175 | + // Index of the largest output element, which is what a classifier wants. |
| 176 | + int argmax(size_t index = 0) { |
| 177 | + const float* values = output(index); |
| 178 | + size_t count = outputCount(index); |
| 179 | + if (values == nullptr || count == 0) { |
| 180 | + return -1; |
| 181 | + } |
| 182 | + int best = 0; |
| 183 | + for (size_t i = 1; i < count; i++) { |
| 184 | + if (values[i] > values[best]) { |
| 185 | + best = static_cast<int>(i); |
| 186 | + } |
| 187 | + } |
| 188 | + return best; |
| 189 | + } |
| 190 | + |
| 191 | + // Why the last call returned false. Never null. |
| 192 | + const char* error() const { |
| 193 | + return error_[0] ? error_ : "no error"; |
| 194 | + } |
| 195 | + |
| 196 | + // For anything this wrapper does not cover. |
| 197 | + executorch::runtime::Program* program() { |
| 198 | + return program_ ? &*program_ : nullptr; |
| 199 | + } |
| 200 | + executorch::runtime::Method* method() { |
| 201 | + return method_ ? &*method_ : nullptr; |
| 202 | + } |
| 203 | + |
| 204 | + private: |
| 205 | + using Span = executorch::runtime::Span<uint8_t>; |
| 206 | + static constexpr size_t kMaxInputs = 4; |
| 207 | + static constexpr size_t kMaxDims = 8; |
| 208 | + |
| 209 | + const executorch::aten::Tensor* outputTensor(size_t index) { |
| 210 | + if (!method_) { |
| 211 | + fail("output before begin()", 0); |
| 212 | + return nullptr; |
| 213 | + } |
| 214 | + auto status = method_->get_outputs(&output_, 1 + index); |
| 215 | + if (status != executorch::runtime::Error::Ok || !output_.isTensor()) { |
| 216 | + fail("get_outputs", static_cast<int>(status)); |
| 217 | + return nullptr; |
| 218 | + } |
| 219 | + out_tensor_.emplace(output_.toTensor()); |
| 220 | + return &*out_tensor_; |
| 221 | + } |
| 222 | + |
| 223 | + // A pool shortfall is by far the most likely cause of a null allocation, and |
| 224 | + // the runtime already logged the exact numbers through et_arduino_log. |
| 225 | + int pool_hint() const { |
| 226 | + return static_cast<int>(allocator_.size()); |
| 227 | + } |
| 228 | + |
| 229 | + bool fail(const char* what, int detail) { |
| 230 | + snprintf(error_, sizeof(error_), "%s failed (0x%x)", what, detail); |
| 231 | + return false; |
| 232 | + } |
| 233 | + |
| 234 | + executorch::extension::BufferDataLoader loader_; |
| 235 | + executorch::runtime::MemoryAllocator allocator_; |
| 236 | + std::optional<executorch::runtime::Program> program_; |
| 237 | + std::optional<executorch::runtime::HierarchicalAllocator> planned_; |
| 238 | + std::optional<executorch::runtime::MemoryManager> manager_; |
| 239 | + std::optional<executorch::runtime::Method> method_; |
| 240 | + const char* method_name_ = nullptr; |
| 241 | + |
| 242 | + std::optional<executorch::aten::TensorImpl> impls_[kMaxInputs]; |
| 243 | + std::optional<executorch::aten::Tensor> tensors_[kMaxInputs]; |
| 244 | + int32_t sizes_[kMaxInputs][kMaxDims] = {}; |
| 245 | + uint8_t dim_order_[kMaxInputs][kMaxDims] = {}; |
| 246 | + |
| 247 | + executorch::runtime::EValue output_; |
| 248 | + std::optional<executorch::aten::Tensor> out_tensor_; |
| 249 | + char error_[96] = {}; |
| 250 | +}; |
0 commit comments