|
| 1 | +/* Copyright 2026 Arm Limited and/or its affiliates. |
| 2 | + * |
| 3 | + * This source code is licensed under the BSD-style license found in the |
| 4 | + * LICENSE file in the root directory of this source tree. |
| 5 | + */ |
| 6 | + |
| 7 | +#include <executorch/extension/data_loader/buffer_data_loader.h> |
| 8 | +#include <executorch/runtime/core/exec_aten/exec_aten.h> |
| 9 | +#include <executorch/runtime/core/exec_aten/util/scalar_type_util.h> |
| 10 | +#include <executorch/runtime/core/memory_allocator.h> |
| 11 | +#include <executorch/runtime/executor/program.h> |
| 12 | +#include <executorch/runtime/platform/log.h> |
| 13 | +#include <executorch/runtime/platform/platform.h> |
| 14 | +#include <executorch/runtime/platform/runtime.h> |
| 15 | +#include <stdio.h> |
| 16 | +#include <algorithm> |
| 17 | +#include <cinttypes> |
| 18 | +#include <vector> |
| 19 | + |
| 20 | +#include "arm_memory_allocator.h" |
| 21 | + |
| 22 | +#if defined(ET_COMPILED_PTE) |
| 23 | +#include "model_pte.h" |
| 24 | +#elif !defined(ET_MODEL_PTE_ADDR) || !defined(ET_MODEL_PTE_SIZE) |
| 25 | +#error "Fixed-address models require ET_MODEL_PTE_ADDR and ET_MODEL_PTE_SIZE" |
| 26 | +#endif |
| 27 | + |
| 28 | +using executorch::aten::ScalarType; |
| 29 | +using executorch::aten::Tensor; |
| 30 | +using executorch::aten::TensorImpl; |
| 31 | +using executorch::extension::BufferDataLoader; |
| 32 | +using executorch::runtime::Error; |
| 33 | +using executorch::runtime::EValue; |
| 34 | +using executorch::runtime::HierarchicalAllocator; |
| 35 | +using executorch::runtime::MemoryManager; |
| 36 | +using executorch::runtime::Method; |
| 37 | +using executorch::runtime::Program; |
| 38 | +using executorch::runtime::Result; |
| 39 | +using executorch::runtime::Span; |
| 40 | + |
| 41 | +#if !defined(ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE) |
| 42 | +#define ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE (4 * 1024 * 1024) |
| 43 | +#endif |
| 44 | + |
| 45 | +// Bare-metal targets do not have virtual memory or a large general-purpose |
| 46 | +// heap. These statically sized arenas make the runtime's memory use explicit. |
| 47 | +// The linker script places the persistent method pool and temporary tensor |
| 48 | +// arena in memory that is accessible to both the CPU and Ethos-U. |
| 49 | +alignas(16) __attribute__((section("input_data_sec"))) unsigned char method_pool |
| 50 | + [ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE]; |
| 51 | +alignas(16) |
| 52 | + __attribute__((section(".bss.tensor_arena"))) unsigned char temp_pool |
| 53 | + [ET_ARM_BAREMETAL_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE]; |
| 54 | + |
| 55 | +// Override the platform logging hook so ExecuTorch diagnostics are visible on |
| 56 | +// the FVP UART (and on the corresponding output device on real hardware). |
| 57 | +[[maybe_unused]] void et_pal_emit_log_message( |
| 58 | + ET_UNUSED et_timestamp_t timestamp, |
| 59 | + et_pal_log_level_t level, |
| 60 | + const char* filename, |
| 61 | + ET_UNUSED const char* function, |
| 62 | + size_t line, |
| 63 | + const char* message, |
| 64 | + ET_UNUSED size_t length) { |
| 65 | + fprintf( |
| 66 | + stderr, |
| 67 | + "%c [executorch:%s:%lu] %s\n", |
| 68 | + level, |
| 69 | + filename, |
| 70 | + static_cast<unsigned long>(line), |
| 71 | + message); |
| 72 | +} |
| 73 | + |
| 74 | +int main() { |
| 75 | + // Runtime initialization registers the platform abstraction before any |
| 76 | + // Program or Method objects are created. |
| 77 | + executorch::runtime::runtime_init(); |
| 78 | + |
| 79 | +#if defined(ET_COMPILED_PTE) |
| 80 | + // The normal example build converts the PTE into a C array and links it into |
| 81 | + // the ELF. sizeof(model_pte) therefore gives the loader an exact bound. |
| 82 | + const uint8_t* model_data = model_pte; |
| 83 | + const size_t model_size = sizeof(model_pte); |
| 84 | +#else |
| 85 | + // A production system can keep a large PTE in flash or another |
| 86 | + // memory-mapped region instead. The address is supplied by the build. |
| 87 | + const uint8_t* model_data = |
| 88 | + reinterpret_cast<const uint8_t*>(ET_MODEL_PTE_ADDR); |
| 89 | + const size_t model_size = ET_MODEL_PTE_SIZE; |
| 90 | +#endif |
| 91 | + |
| 92 | + // BufferDataLoader lets Program read the serialized PTE directly from its |
| 93 | + // current memory location, without first copying the whole model. |
| 94 | + BufferDataLoader loader(model_data, model_size); |
| 95 | + Result<Program> program = Program::load(&loader); |
| 96 | + ET_CHECK_MSG(program.ok(), "Program::load failed: 0x%x", program.error()); |
| 97 | + |
| 98 | + // Classic ML exports normally contain a single inference method. Reading |
| 99 | + // its metadata first tells us how much planned memory must be provided |
| 100 | + // before the method itself can be loaded. |
| 101 | + const auto method_name = program->get_method_name(0); |
| 102 | + ET_CHECK_MSG(method_name.ok(), "Program has no methods"); |
| 103 | + const auto method_meta = program->method_meta(*method_name); |
| 104 | + ET_CHECK_MSG(method_meta.ok(), "Could not load method metadata"); |
| 105 | + |
| 106 | + // Persistent allocations, including method state and input storage, come |
| 107 | + // from method_pool. Temporary kernel allocations come from temp_pool and |
| 108 | + // only need to remain valid during execute(). |
| 109 | + ArmMemoryAllocator method_allocator(sizeof(method_pool), method_pool); |
| 110 | + ArmMemoryAllocator temp_allocator(sizeof(temp_pool), temp_pool); |
| 111 | + |
| 112 | + // The exporter memory-plans intermediate and output tensors into one or more |
| 113 | + // numbered buffers. Allocate every requested buffer from the persistent |
| 114 | + // arena and preserve the spans for the lifetime of the loaded Method. |
| 115 | + std::vector<Span<uint8_t>> planned_spans; |
| 116 | + planned_spans.reserve(method_meta->num_memory_planned_buffers()); |
| 117 | + for (size_t i = 0; i < method_meta->num_memory_planned_buffers(); ++i) { |
| 118 | + const size_t size = method_meta->memory_planned_buffer_size(i).get(); |
| 119 | + auto* buffer = static_cast<uint8_t*>(method_allocator.allocate(size, 16)); |
| 120 | + ET_CHECK_MSG(buffer != nullptr, "Could not allocate planned buffer %lu", i); |
| 121 | + planned_spans.push_back({buffer, size}); |
| 122 | + } |
| 123 | + |
| 124 | + // HierarchicalAllocator maps the PTE's planned-buffer IDs to the spans above. |
| 125 | + // MemoryManager combines planned, persistent, and temporary allocation into |
| 126 | + // the three memory classes used while loading and executing a method. |
| 127 | + HierarchicalAllocator planned_memory( |
| 128 | + {planned_spans.data(), planned_spans.size()}); |
| 129 | + MemoryManager memory_manager( |
| 130 | + &method_allocator, &planned_memory, &temp_allocator); |
| 131 | + |
| 132 | + // Keep Program, all allocators, planned_spans, and Method alive until after |
| 133 | + // inference. The Method retains references to memory owned by these objects. |
| 134 | + Result<Method> method = program->load_method(*method_name, &memory_manager); |
| 135 | + ET_CHECK_MSG(method.ok(), "Could not load method: 0x%x", method.error()); |
| 136 | + |
| 137 | + // Graph inputs are caller-owned and are not generally backed by the |
| 138 | + // memory-planned buffers. Allocate each input from method_pool, construct a |
| 139 | + // tensor with the PTE's exact shape and scalar type, fill it with |
| 140 | + // deterministic ones, and bind it with set_input(). Using the method arena |
| 141 | + // rather than malloc is important because the bare-metal libc heap is |
| 142 | + // intentionally small and cannot hold common image inputs such as |
| 143 | + // MobileNetV2's. |
| 144 | + const auto loaded_method_meta = method->method_meta(); |
| 145 | + for (size_t i = 0; i < loaded_method_meta.num_inputs(); ++i) { |
| 146 | + const auto input_tag = loaded_method_meta.input_tag(i); |
| 147 | + ET_CHECK_MSG( |
| 148 | + input_tag.ok(), |
| 149 | + "Could not read input %lu metadata", |
| 150 | + static_cast<unsigned long>(i)); |
| 151 | + ET_CHECK_MSG( |
| 152 | + input_tag.get() == executorch::runtime::Tag::Tensor, |
| 153 | + "Input %lu is not a tensor", |
| 154 | + static_cast<unsigned long>(i)); |
| 155 | + |
| 156 | + auto tensor_meta = loaded_method_meta.input_tensor_meta(i); |
| 157 | + ET_CHECK_MSG( |
| 158 | + tensor_meta.ok(), |
| 159 | + "Could not read input %lu tensor metadata", |
| 160 | + static_cast<unsigned long>(i)); |
| 161 | + void* input_buffer = method_allocator.allocate(tensor_meta->nbytes(), 16); |
| 162 | + ET_CHECK_MSG( |
| 163 | + input_buffer != nullptr, |
| 164 | + "Could not allocate input %lu", |
| 165 | + static_cast<unsigned long>(i)); |
| 166 | + TensorImpl input_impl( |
| 167 | + tensor_meta->scalar_type(), |
| 168 | + tensor_meta->sizes().size(), |
| 169 | + const_cast<TensorImpl::SizesType*>(tensor_meta->sizes().data()), |
| 170 | + input_buffer, |
| 171 | + const_cast<TensorImpl::DimOrderType*>(tensor_meta->dim_order().data())); |
| 172 | + Tensor input(&input_impl); |
| 173 | + switch (input.scalar_type()) { |
| 174 | +#define FILL_INPUT(cpp_type, scalar_type) \ |
| 175 | + case ScalarType::scalar_type: { \ |
| 176 | + cpp_type* data = input.mutable_data_ptr<cpp_type>(); \ |
| 177 | + std::fill(data, data + input.numel(), cpp_type(1)); \ |
| 178 | + break; \ |
| 179 | + } |
| 180 | + ET_FORALL_REALHBBF16_TYPES(FILL_INPUT) |
| 181 | +#undef FILL_INPUT |
| 182 | + default: |
| 183 | + ET_CHECK_MSG(false, "Unsupported input tensor type"); |
| 184 | + } |
| 185 | + ET_CHECK_MSG( |
| 186 | + method->set_input(input, i) == Error::Ok, |
| 187 | + "Could not prepare input %lu", |
| 188 | + static_cast<unsigned long>(i)); |
| 189 | + } |
| 190 | + |
| 191 | + // All input and planned-buffer storage is now bound, so the method can safely |
| 192 | + // execute in place. Delegate calls run on Ethos-U and any remaining |
| 193 | + // operators run on the CPU. |
| 194 | + ET_CHECK_MSG(method->execute() == Error::Ok, "Inference failed"); |
| 195 | + |
| 196 | + // Output EValues reference tensors in planned memory. They do not own or copy |
| 197 | + // the tensor data, so consume them while Method and planned_memory are alive. |
| 198 | + std::vector<EValue> outputs(method->outputs_size()); |
| 199 | + ET_CHECK_MSG( |
| 200 | + method->get_outputs(outputs.data(), outputs.size()) == Error::Ok, |
| 201 | + "Could not get outputs"); |
| 202 | + ET_LOG( |
| 203 | + Info, |
| 204 | + "Inference complete: %lu output(s)", |
| 205 | + static_cast<unsigned long>(outputs.size())); |
| 206 | + for (size_t output_index = 0; output_index < outputs.size(); ++output_index) { |
| 207 | + ET_CHECK_MSG(outputs[output_index].isTensor(), "Output is not a tensor"); |
| 208 | + Tensor output = outputs[output_index].toTensor(); |
| 209 | + |
| 210 | + // Print every element so this minimal runner is useful with classification |
| 211 | + // logits as well as small integer or boolean test models. Tensor storage is |
| 212 | + // interpreted according to ScalarType rather than as untyped bytes. |
| 213 | + for (size_t element_index = 0; element_index < output.numel(); |
| 214 | + ++element_index) { |
| 215 | + printf( |
| 216 | + "Output[%lu][%lu]: ", |
| 217 | + static_cast<unsigned long>(output_index), |
| 218 | + static_cast<unsigned long>(element_index)); |
| 219 | + switch (output.scalar_type()) { |
| 220 | + case ScalarType::Byte: |
| 221 | + printf( |
| 222 | + "%u\n", |
| 223 | + static_cast<unsigned int>( |
| 224 | + output.const_data_ptr<uint8_t>()[element_index])); |
| 225 | + break; |
| 226 | + case ScalarType::Char: |
| 227 | + printf("%d\n", output.const_data_ptr<int8_t>()[element_index]); |
| 228 | + break; |
| 229 | + case ScalarType::Short: |
| 230 | + printf("%d\n", output.const_data_ptr<int16_t>()[element_index]); |
| 231 | + break; |
| 232 | + case ScalarType::Int: |
| 233 | + printf("%d\n", output.const_data_ptr<int32_t>()[element_index]); |
| 234 | + break; |
| 235 | + case ScalarType::Long: |
| 236 | + printf( |
| 237 | + "%" PRId64 "\n", output.const_data_ptr<int64_t>()[element_index]); |
| 238 | + break; |
| 239 | + case ScalarType::Half: |
| 240 | + printf( |
| 241 | + "%f\n", |
| 242 | + static_cast<double>(output.const_data_ptr< |
| 243 | + executorch::aten::Half>()[element_index])); |
| 244 | + break; |
| 245 | + case ScalarType::Float: |
| 246 | + printf("%f\n", output.const_data_ptr<float>()[element_index]); |
| 247 | + break; |
| 248 | + case ScalarType::Double: |
| 249 | + printf("%f\n", output.const_data_ptr<double>()[element_index]); |
| 250 | + break; |
| 251 | + case ScalarType::Bool: |
| 252 | + printf( |
| 253 | + "%s\n", |
| 254 | + output.const_data_ptr<bool>()[element_index] ? "true" : "false"); |
| 255 | + break; |
| 256 | + case ScalarType::BFloat16: |
| 257 | + printf( |
| 258 | + "%f\n", |
| 259 | + static_cast<double>( |
| 260 | + output.const_data_ptr< |
| 261 | + executorch::aten::BFloat16>()[element_index])); |
| 262 | + break; |
| 263 | + default: |
| 264 | + ET_CHECK_MSG(false, "Unsupported output tensor type"); |
| 265 | + } |
| 266 | + } |
| 267 | + } |
| 268 | + |
| 269 | + // ASCII EOT asks the FVP UART model to stop the simulation cleanly. |
| 270 | + printf("\04"); |
| 271 | + return 0; |
| 272 | +} |
0 commit comments