Skip to content

Commit 3546e32

Browse files
authored
Arm backend: Support Ethos-U85 KV-cache export (pytorch#21913)
- Add the export/runtime plumbing needed for KV-cache SmolLM2-style models on Ethos-U. This includes int32 token/input_pos example inputs for Arm backends, initialized KV cache mutable buffers, calibration dtype preservation, and an Ethos-U partitioner guard for unsupported KV cache index ops. - Extend the semihosting runner with input rebinding and server mode so step-wise KV-cache decoding can reuse one FVP process across token runs. AI-assisted-by: Codex cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Xingguo Li <xingguo.li@arm.com>
1 parent 3ca32af commit 3546e32

4 files changed

Lines changed: 155 additions & 17 deletions

File tree

‎examples/arm/executor_runner/arm_executor_runner.cpp‎

Lines changed: 102 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ unsigned char __attribute__((
193193

194194
using executorch::aten::ScalarType;
195195
using executorch::aten::Tensor;
196+
using executorch::aten::TensorImpl;
196197
using executorch::extension::BufferDataLoader;
197198
using executorch::runtime::Error;
198199
using executorch::runtime::EValue;
@@ -499,12 +500,19 @@ Error prepare_input_tensors(
499500
tensor_size);
500501
err = Error::InvalidArgument;
501502
} else if (input_evalues[i].isTensor()) {
502-
// Copy the data from the input buffer to the tensor
503-
Tensor& tensor = input_evalues[i].toTensor();
504-
std::memcpy( // NOLINT(memcpy) - Buffer size is checked
505-
tensor.mutable_data_ptr<int8_t>(),
503+
// set_input copies shape metadata into its method-owned input tensor.
504+
// For unplanned inputs it aliases only buffer, which remains valid
505+
// through execute(); this temporary TensorImpl is not retained.
506+
TensorImpl impl = TensorImpl(
507+
tensor_meta->scalar_type(),
508+
static_cast<ssize_t>(tensor_meta->sizes().size()),
509+
const_cast<TensorImpl::SizesType*>(tensor_meta->sizes().data()),
506510
buffer,
507-
buffer_size);
511+
const_cast<TensorImpl::DimOrderType*>(
512+
tensor_meta->dim_order().data()));
513+
Tensor tensor(&impl);
514+
err = method.set_input(tensor, i);
515+
ET_CHECK_OK_OR_RETURN_ERROR(err);
508516
}
509517
}
510518

@@ -609,9 +617,34 @@ struct RunnerContext {
609617
#if defined(SEMIHOSTING)
610618
Box<ArmMemoryAllocator> input_file_allocator;
611619
const char* output_basename = nullptr;
620+
bool server_mode = false;
621+
std::vector<const char*> input_filenames;
612622
#endif
613623
};
614624

625+
#if defined(SEMIHOSTING)
626+
Error read_input_files(
627+
RunnerContext& ctx,
628+
const std::vector<const char*>& input_filenames,
629+
std::vector<std::pair<char*, size_t>>& input_buffers) {
630+
input_buffers.clear();
631+
for (size_t i = 0; i < input_filenames.size(); ++i) {
632+
auto [buffer, buffer_size] =
633+
read_binary_file(input_filenames[i], ctx.input_file_allocator.value());
634+
if (buffer == nullptr) {
635+
ET_LOG(
636+
Error,
637+
"Reading input tensor %zu from file %s failed.",
638+
i + 1,
639+
input_filenames[i]);
640+
return Error::AccessFailed;
641+
}
642+
input_buffers.push_back(std::make_pair(buffer, buffer_size));
643+
}
644+
return Error::Ok;
645+
}
646+
#endif
647+
615648
void runner_init(
616649
RunnerContext& ctx,
617650
const uint8_t* model_data,
@@ -1324,6 +1357,48 @@ bool run_model(RunnerContext& ctx, const void* model_data) {
13241357
return model_ok;
13251358
}
13261359

1360+
#if defined(SEMIHOSTING)
1361+
bool run_model_server(
1362+
RunnerContext& ctx,
1363+
const void* model_data,
1364+
std::vector<std::pair<char*, size_t>>& input_buffers) {
1365+
ET_LOG(Info, "Running in semihosting server mode.");
1366+
char line[16];
1367+
int server_inference_count = 0;
1368+
while (fgets(line, sizeof(line), stdin) != nullptr) {
1369+
ctx.input_file_allocator.reset(
1370+
input_file_allocation_pool_size, input_file_allocation_pool);
1371+
Error status = read_input_files(ctx, ctx.input_filenames, input_buffers);
1372+
ET_CHECK_MSG(
1373+
status == Error::Ok, "Could not reload inputs: 0x%" PRIx32, status);
1374+
1375+
status = ::prepare_input_tensors(
1376+
*ctx.method.value(), ctx.temp_allocator.value(), input_buffers);
1377+
ET_CHECK_MSG(
1378+
status == Error::Ok, "Failed to prepare inputs 0x%" PRIx32, status);
1379+
1380+
StartMeasurements();
1381+
status = ctx.method.value()->execute();
1382+
StopMeasurements(1);
1383+
ET_CHECK_MSG(
1384+
status == Error::Ok,
1385+
"Execution of method %s failed with status 0x%" PRIx32,
1386+
ctx.method_name,
1387+
status);
1388+
1389+
print_outputs(ctx);
1390+
ctx.temp_allocator.reset(temp_allocation_pool_size, temp_allocation_pool);
1391+
ET_LOG(Info, "SERVER_INFERENCE_DONE %d", server_inference_count++);
1392+
fflush(stdout);
1393+
fflush(stderr);
1394+
}
1395+
1396+
bool model_ok = verify_result(ctx, model_data);
1397+
ET_LOG(Info, "Model run: %d", model_ok);
1398+
return model_ok;
1399+
}
1400+
#endif
1401+
13271402
} // namespace
13281403

13291404
int main(int argc, const char* argv[]) {
@@ -1399,6 +1474,7 @@ int main(int argc, const char* argv[]) {
13991474
_exit(1);
14001475
}
14011476
input_buffers.push_back(std::make_pair(buffer, buffer_size));
1477+
ctx.input_filenames.push_back(input_tensor_filename);
14021478
} else if (std::strcmp(argv[i], "-m") == 0) {
14031479
const char* pte_filename = argv[++i];
14041480
ET_LOG(Info, "Reading pte model from file %s", pte_filename);
@@ -1414,6 +1490,19 @@ int main(int argc, const char* argv[]) {
14141490
} else if (std::strcmp(argv[i], "-o") == 0) {
14151491
// store the base filename to write output to.
14161492
ctx.output_basename = argv[++i];
1493+
} else if (std::strcmp(argv[i], "--server_mode") == 0) {
1494+
if (++i >= argc) {
1495+
ET_LOG(Fatal, "--server_mode requires true or false");
1496+
return 1;
1497+
}
1498+
if (std::strcmp(argv[i], "true") == 0) {
1499+
ctx.server_mode = true;
1500+
} else if (std::strcmp(argv[i], "false") == 0) {
1501+
ctx.server_mode = false;
1502+
} else {
1503+
ET_LOG(Fatal, "Invalid --server_mode value: %s", argv[i]);
1504+
return 1;
1505+
}
14171506
}
14181507
}
14191508
#endif
@@ -1442,7 +1531,15 @@ int main(int argc, const char* argv[]) {
14421531

14431532
runner_init(ctx, model_data, model_size, input_buffers);
14441533
bool model_ok = true;
1534+
#if defined(SEMIHOSTING)
1535+
if (ctx.server_mode) {
1536+
model_ok = run_model_server(ctx, model_data, input_buffers);
1537+
} else {
1538+
model_ok = run_model(ctx, model_data);
1539+
}
1540+
#else
14451541
model_ok = run_model(ctx, model_data);
1542+
#endif
14461543
ET_LOG(Info, "Model run: %d", model_ok);
14471544

14481545
log_mem_status(ctx);

‎examples/models/llama/export_llama_lib.py‎

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
from executorch.extension.llm.export.config.llm_config import LlmConfig
3636
from executorch.extension.llm.export.partitioner_lib import (
3737
get_coreml_partitioner,
38-
get_ethosu_partitioner,
3938
get_mps_partitioner,
4039
get_openvino_partitioner,
4140
get_qnn_partitioner,
@@ -1263,12 +1262,41 @@ def _to_edge_and_lower_llama_arm(
12631262

12641263
partitioners = []
12651264
if llm_config.backend.ethosu.enabled:
1265+
from executorch.backends.arm.ethosu.compile_spec import EthosUCompileSpec
1266+
from executorch.backends.arm.ethosu.partitioner import EthosUPartitioner
1267+
1268+
compile_spec = EthosUCompileSpec(
1269+
llm_config.backend.ethosu.target,
1270+
system_config=(
1271+
None
1272+
if llm_config.backend.ethosu.system_config == "default"
1273+
else llm_config.backend.ethosu.system_config
1274+
),
1275+
memory_mode=(
1276+
None
1277+
if llm_config.backend.ethosu.memory_mode == "default"
1278+
else llm_config.backend.ethosu.memory_mode
1279+
),
1280+
extra_flags=llm_config.backend.ethosu.extra_flags,
1281+
)
1282+
1283+
additional_checks = []
1284+
if (
1285+
llm_config.model.use_kv_cache
1286+
and not llm_config.model.static_quantize_kv_cache
1287+
):
1288+
from torch.fx.passes.operator_support import OperatorSupportBase
1289+
1290+
class EthosUKVCacheOperatorSupport(OperatorSupportBase):
1291+
def is_node_supported(self, submodules, node):
1292+
return "aten.index.Tensor" not in str(node.target)
1293+
1294+
additional_checks.append(EthosUKVCacheOperatorSupport())
1295+
12661296
partitioners.append(
1267-
get_ethosu_partitioner(
1268-
llm_config.backend.ethosu.target,
1269-
llm_config.backend.ethosu.system_config,
1270-
llm_config.backend.ethosu.memory_mode,
1271-
llm_config.backend.ethosu.extra_flags,
1297+
EthosUPartitioner(
1298+
compile_spec,
1299+
additional_checks=additional_checks or None,
12721300
)
12731301
)
12741302
modelname = f"ethosu_{modelname}"

‎examples/models/llama/model.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -308,19 +308,25 @@ def get_example_inputs(self):
308308

309309
# assumption is the custom op doesnt support dynamic shape right now. It might but its untested so lets first get static shape working
310310
def get_example_inputs_kvcache_sdpa(self):
311+
backend = self.llm_config.backend
312+
token_dtype = (
313+
torch.int32
314+
if (backend.ethosu.enabled or backend.tosa.enabled or backend.vgf.enabled)
315+
else torch.long
316+
)
311317
if self.enable_dynamic_shape:
312318
return (
313-
torch.tensor([[2, 3, 4]], dtype=torch.long),
314-
{"input_pos": torch.tensor([0], dtype=torch.long)},
319+
torch.tensor([[2, 3, 4]], dtype=token_dtype),
320+
{"input_pos": torch.tensor([0], dtype=token_dtype)},
315321
)
316322
else:
317323
return (
318324
torch.tensor(
319-
[[1]], dtype=torch.long
325+
[[1]], dtype=token_dtype
320326
), # tokens, with kv cache our input token length is always just 1 token.
321327
{
322328
"input_pos": torch.tensor(
323-
[0], dtype=torch.long
329+
[0], dtype=token_dtype
324330
) # start_pos, what token of output are we on.
325331
},
326332
)

‎extension/llm/export/builder.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,9 +311,16 @@ def calibrate_template(
311311
while token_list[-1] != tokenizer.eos_id and pos < max_len:
312312
logits_token_pos = -1
313313
if self.use_kv_cache:
314+
input_dtype = self.example_inputs[0].dtype
315+
if self.example_kwarg_inputs is not None:
316+
input_pos_dtype = self.example_kwarg_inputs[
317+
"input_pos"
318+
].dtype
319+
else:
320+
input_pos_dtype = self.example_inputs[1]["input_pos"].dtype
314321
logits = module(
315-
torch.full((1, 1), token_list[pos]),
316-
{"input_pos": torch.tensor((pos,))},
322+
torch.full((1, 1), token_list[pos], dtype=input_dtype),
323+
{"input_pos": torch.tensor((pos,), dtype=input_pos_dtype)},
317324
)
318325
else:
319326
prefix, logits_token_pos = self._prepare_calibration_prefix(

0 commit comments

Comments
 (0)