Skip to content

Commit 713e159

Browse files
authored
Arm backend: Support memory planning in arm_executor_runner (pytorch#21774)
Support explicitly tagging tensors with mem_id = 2 for putting them in a SLOW_MEMORY_REGION and mem_id = 3 for putting in FAST_MEMORY_REGION. Keep default mem_id = 1 in SLOW_MEMORY_REGION. This gives users more flexibility in memory planning their programs. Add a new .fast_memory input section in .sram.bss that backs the fast memory allocator. The size of the memory area is set by a new cmake flag ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE. The slow memory region can still be allocated like planned et tensors are currently. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Erik Lundell <erik.lundell@arm.com>
1 parent ce57229 commit 713e159

7 files changed

Lines changed: 217 additions & 8 deletions

File tree

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import argparse
7+
from pathlib import Path
8+
from typing import Optional
9+
10+
import executorch.exir as exir
11+
import torch
12+
from executorch.exir import to_edge
13+
from executorch.exir.memory_planning import greedy, MemoryPlanningAlgorithmSuite
14+
from executorch.exir.pass_base import PassResult
15+
from executorch.exir.passes import MemoryPlanningPass
16+
from torch.export import export
17+
from torch.export.exported_program import ExportGraphSignature
18+
from torch.fx import GraphModule
19+
20+
21+
class CustomPoolMemoryPlanningPass(MemoryPlanningPass):
22+
def call(self, graph_module: GraphModule) -> PassResult:
23+
for subgm in graph_module.modules():
24+
if not isinstance(subgm, GraphModule):
25+
continue
26+
for node in subgm.graph.nodes:
27+
if node.op == "placeholder":
28+
node.meta["spec"].mem_id = 1
29+
continue
30+
31+
if node.op != "call_function":
32+
continue
33+
34+
if node.target == torch.ops.aten.add.out:
35+
node.meta["spec"].mem_id = 3
36+
elif node.target == torch.ops.aten.mul.out:
37+
node.meta["spec"].mem_id = 1
38+
39+
return super().run(graph_module)
40+
41+
def run(
42+
self,
43+
graph_module: GraphModule,
44+
graph_signature: Optional[ExportGraphSignature] = None,
45+
) -> PassResult:
46+
return self.call(graph_module)
47+
48+
49+
class MultiplePoolsToyModel(torch.nn.Module):
50+
def forward(self, a: torch.Tensor) -> torch.Tensor:
51+
b = a + a
52+
c = a * b
53+
d = c + b
54+
e = c * d
55+
return e
56+
57+
58+
def export_pte(output_path: Path) -> None:
59+
edge_program = to_edge(
60+
export(MultiplePoolsToyModel(), (torch.ones(1),), strict=True)
61+
)
62+
mem_algo = MemoryPlanningAlgorithmSuite(algo_list=[greedy])
63+
executorch_program = edge_program.to_executorch(
64+
exir.ExecutorchBackendConfig(
65+
memory_planning_pass=CustomPoolMemoryPlanningPass(
66+
memory_planning_algo=mem_algo,
67+
alignment=1,
68+
),
69+
)
70+
)
71+
72+
output_path.parent.mkdir(parents=True, exist_ok=True)
73+
with output_path.open("wb") as output_file:
74+
executorch_program.write_to_file(output_file)
75+
76+
77+
def main() -> None:
78+
parser = argparse.ArgumentParser(
79+
description="Export a portable .pte with mem_id=3 planned tensor storage."
80+
)
81+
parser.add_argument(
82+
"--output",
83+
required=True,
84+
type=Path,
85+
help="Path to the .pte file to write.",
86+
)
87+
args = parser.parse_args()
88+
89+
export_pte(args.output)
90+
91+
92+
if __name__ == "__main__":
93+
main()

‎backends/arm/test/test_arm_backend.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,32 @@ test_memory_allocation() {
484484
--require "Total DRAM used" "<= 0.06 KiB"
485485

486486
_test_runner_size_optimization
487+
488+
echo "${TEST_SUITE_NAME}: Test planned slow and fast memory allocation"
489+
local plan_dir="arm_test/test_run/memory_planning"
490+
local pte_file="${plan_dir}/memory_planning_mem_id_3.pte"
491+
local runner_dir="${plan_dir}/cmake-out"
492+
local planned_log="${plan_dir}/planned.log"
493+
mkdir -p "${plan_dir}"
494+
495+
python3 "${et_root_dir}/backends/arm/test/assets/export_memory_planning_mem_id_3.py" \
496+
--output="${pte_file}"
497+
498+
backends/arm/scripts/build_executor_runner.sh \
499+
--pte="${pte_file}" \
500+
--target=ethos-u55-128 \
501+
--output="${runner_dir}" \
502+
--select_ops_list="aten::add.out,aten::mul.out" \
503+
--extra_build_flags="-DFETCH_ETHOS_U_CONTENT=OFF -DET_ARM_BAREMETAL_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE=0x180000 -DET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE=0x1000"
504+
505+
backends/arm/scripts/run_fvp.sh \
506+
--elf="${runner_dir}/arm_executor_runner" \
507+
--target=ethos-u55-128 \
508+
&> "${planned_log}"
509+
510+
python3 backends/arm/test/test_memory_allocator_log.py --log "${planned_log}" \
511+
--require "method_allocator_planned" "== 8 B" \
512+
--require "planned_fast_used" "== 8 B"
487513
echo "${TEST_SUITE_NAME}: PASS"
488514
}
489515

‎examples/arm/ethos-u-porting-guide.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ way, depending on the `ETHOSU_ARENA` parameter, the linker knows whether the inp
8989
its value is derived based on the memory mode parameter that is passed to the `examples/arm/run.sh` shell script. Then, at link time, the .ddr.bss section is always placed in the external memory and the .sram.bss is always placed in the SRAM.
9090
Finally, note that in the `examples/arm/executor_runner/arm_executor_runner.cpp` application code, we place the buffers for the Ethos-u scratch and the neural network in the correct symbol from the linker script. For instance,
9191
the Ethos-u scratch buffer corresponds to the the `.bss.tensor_arena` input section in the linker script, In the application code, when we allocate memory for the Ethos-u scratch buffer, we place this array in the .bss.tensor_arena section in the memory map.
92+
The same runner also supports ExecuTorch memory-planned tensors in `SLOW_MEMORY_REGION` for `mem_id=1` and `mem_id=2`, and in `FAST_MEMORY_REGION` for `mem_id=3`, backed by the `.fast_memory` input section in `.sram.bss` and sized with `ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE`.
9293

9394
```
9495
unsigned char __attribute__((

‎examples/arm/executor_runner/CMakeLists.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,11 @@ option(
101101
"Set ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE to specify memory alloction pool size"
102102
OFF
103103
)
104+
set(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE
105+
""
106+
CACHE STRING
107+
"Size of the SRAM-backed planned fast memory pool used for mem_id=3"
108+
)
104109

105110
if(NOT DEFINED PYTHON_EXECUTABLE)
106111
find_package(
@@ -202,6 +207,10 @@ message(
202207
STATUS
203208
"ET_ARM_BAREMETAL_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE = ${ET_ARM_BAREMETAL_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE}"
204209
)
210+
message(
211+
STATUS
212+
"ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE = ${ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE}"
213+
)
205214

206215
# Dependencies from the Ethos-U Core This is the platform target of
207216
# Corstone-300, that includes ethosu_core_driver and bare-metal bringup
@@ -679,6 +688,14 @@ if(DEFINED ET_ARM_BAREMETAL_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE)
679688
)
680689
endif()
681690

691+
if(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE)
692+
target_compile_definitions(
693+
arm_executor_runner
694+
PUBLIC
695+
ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE=${ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE}
696+
)
697+
endif()
698+
682699
# Fixup compilation of retarget.c
683700
if(SEMIHOSTING)
684701
# Remove this when MLBEDSW-8910 is closed.

‎examples/arm/executor_runner/Corstone-300.ld‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ SECTIONS
229229
. = ALIGN(32);
230230
*(.bss.tensor_arena)
231231
#endif
232+
. = ALIGN(16);
233+
*(.fast_memory)
232234
. = ALIGN(16);
233235
*(.bss.ethosu_scratch);
234236
} > SRAM :null

‎examples/arm/executor_runner/Corstone-320.ld‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,8 @@ SECTIONS
235235
. = ALIGN(32);
236236
*(.bss.tensor_arena)
237237
#endif
238+
. = ALIGN(16);
239+
*(.fast_memory)
238240
. = ALIGN(16);
239241
*(.bss.ethosu_scratch);
240242
} > SRAM :null

‎examples/arm/executor_runner/arm_executor_runner.cpp‎

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@
9494
* ET_ARM_BAREMETAL_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE - Size of memory area
9595
* used when running
9696
* inferences
97+
* ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE - Size of the fast
98+
* planned memory area
99+
* for mem_id = 3
97100
*/
98101

99102
#include <errno.h>
@@ -287,6 +290,15 @@ unsigned char* ethosu_fast_scratch = dedicated_sram;
287290
}
288291
#endif
289292

293+
#if defined(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE)
294+
const size_t planned_fast_memory_pool_size =
295+
ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE;
296+
unsigned char __attribute__((
297+
section(".fast_memory"),
298+
aligned(16))) planned_fast_memory_pool[planned_fast_memory_pool_size];
299+
#endif
300+
constexpr size_t FAST_MEMORY_REGION_INDEX = 2;
301+
290302
[[maybe_unused]] void et_pal_init(void) {
291303
#if defined(__ARM_ARCH_8_1M_MAIN__)
292304
// Armv8.1-M Mainline cores (M55, M85) have the optional PMU extension.
@@ -582,6 +594,7 @@ struct RunnerContext {
582594
Box<BufferDataLoader> loader;
583595
Box<Program> program;
584596
Box<ArmMemoryAllocator> method_allocator;
597+
Box<ArmMemoryAllocator> planned_fast_allocator;
585598
Box<ArmMemoryAllocator> temp_allocator;
586599
std::vector<Span<uint8_t>> planned_spans;
587600
Box<HierarchicalAllocator> planned_memory;
@@ -673,6 +686,15 @@ void runner_init(
673686
ctx.method_allocator.reset(
674687
method_allocation_pool_size, method_allocation_pool);
675688

689+
#if defined(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE)
690+
ET_LOG(
691+
Info,
692+
"Setup planned FAST_MEMORY_REGION pool. Size: %lu bytes.",
693+
static_cast<unsigned long>(planned_fast_memory_pool_size));
694+
ctx.planned_fast_allocator.reset(
695+
planned_fast_memory_pool_size, planned_fast_memory_pool);
696+
#endif
697+
676698
ctx.planned_spans.clear();
677699
size_t num_memory_planned_buffers = method_meta->num_memory_planned_buffers();
678700
ctx.planned_spans.reserve(num_memory_planned_buffers);
@@ -681,20 +703,49 @@ void runner_init(
681703
for (size_t id = 0; id < num_memory_planned_buffers; ++id) {
682704
size_t buffer_size =
683705
static_cast<size_t>(method_meta->memory_planned_buffer_size(id).get());
706+
684707
ET_LOG(
685708
Info,
686-
"Setting up planned buffer %lu, size %lu.",
687-
static_cast<unsigned long>(id),
709+
"Setting up planned buffer with mem_id=%lu, size %lu.",
710+
static_cast<unsigned long>(id + 1),
688711
static_cast<unsigned long>(buffer_size));
712+
if (buffer_size == 0) {
713+
ctx.planned_spans.push_back(Span<uint8_t>(
714+
static_cast<uint8_t*>(nullptr), static_cast<size_t>(0)));
715+
continue;
716+
}
689717

690-
/* Move to it's own allocator when MemoryPlanner is in place. */
691-
/* Ethos-U driver requires 16 bit alignment. */
692-
uint8_t* buffer = reinterpret_cast<uint8_t*>(
693-
ctx.method_allocator->allocate(buffer_size, 16UL));
718+
uint8_t* buffer = nullptr;
719+
const char* memory_region = nullptr;
720+
switch (id) {
721+
case FAST_MEMORY_REGION_INDEX:
722+
memory_region = "FAST_MEMORY_REGION";
723+
#if defined(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE)
724+
buffer = reinterpret_cast<uint8_t*>(
725+
ctx.planned_fast_allocator->allocate(buffer_size, 16UL));
726+
#else
727+
ET_CHECK_MSG(
728+
false,
729+
"Planned buffer %lu uses mem_id=%lu/FAST_MEMORY_REGION, but "
730+
"ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE is not set",
731+
static_cast<unsigned long>(id),
732+
static_cast<unsigned long>(id + 1));
733+
#endif
734+
break;
735+
default:
736+
memory_region = "SLOW_MEMORY_REGION";
737+
/* Ethos-U driver requires 16 bit alignment. */
738+
buffer = reinterpret_cast<uint8_t*>(
739+
ctx.method_allocator->allocate(buffer_size, 16UL));
740+
break;
741+
}
694742
ET_CHECK_MSG(
695743
buffer != nullptr,
696-
"Could not allocate memory for memory planned buffer size %lu",
697-
static_cast<unsigned long>(buffer_size));
744+
"Could not allocate memory for planned buffer with mem_id=%lu, size "
745+
"%lu in %s",
746+
static_cast<unsigned long>(id + 1),
747+
static_cast<unsigned long>(buffer_size),
748+
memory_region);
698749
ctx.planned_spans.push_back({buffer, buffer_size});
699750
}
700751

@@ -917,6 +968,23 @@ void log_mem_status(RunnerContext& ctx) {
917968
Info,
918969
"method_allocator_planned: %lu bytes",
919970
static_cast<unsigned long>(ctx.planned_buffer_memsize));
971+
#if defined(ET_ARM_BAREMETAL_PLANNED_FAST_MEMORY_SIZE)
972+
if (ctx.planned_fast_allocator->size() > 0) {
973+
ET_LOG(
974+
Info,
975+
"planned_fast_allocator: %lu / %lu free: %lu ( used: %lu %% ) ",
976+
static_cast<unsigned long>(ctx.planned_fast_allocator->used_size()),
977+
static_cast<unsigned long>(ctx.planned_fast_allocator->size()),
978+
static_cast<unsigned long>(ctx.planned_fast_allocator->free_size()),
979+
static_cast<unsigned long>(
980+
100 * ctx.planned_fast_allocator->used_size() /
981+
ctx.planned_fast_allocator->size()));
982+
ET_LOG(
983+
Info,
984+
"planned_fast_used: %lu bytes",
985+
static_cast<unsigned long>(ctx.planned_fast_allocator->used_size()));
986+
}
987+
#endif
920988
ET_LOG(
921989
Info,
922990
"method_allocator_loaded: %lu bytes",

0 commit comments

Comments
 (0)