Skip to content

Commit 8972b69

Browse files
authored
1 parent 89341d7 commit 8972b69

5 files changed

Lines changed: 241 additions & 11 deletions

File tree

‎.ci/scripts/test_huggingface_optimum_model.py‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
from datasets import load_dataset
1212

1313
from optimum.executorch import (
14-
ExecuTorchModelForCausalLM,
1514
ExecuTorchModelForImageClassification,
1615
ExecuTorchModelForMaskedLM,
1716
ExecuTorchModelForSeq2SeqLM,
@@ -152,18 +151,24 @@ def test_text_generation(model_id, model_dir, recipe, *, quantize=True, run_only
152151
cli_export(command, model_dir)
153152

154153
tokenizer = AutoTokenizer.from_pretrained(model_id)
155-
tokenizer.save_pretrained(model_dir)
156-
model = ExecuTorchModelForCausalLM.from_pretrained(model_dir)
157-
generated_text = model.text_generation(
158-
tokenizer=tokenizer,
159-
prompt="Simply put, the theory of relativity states that",
160-
max_seq_len=64,
154+
saved_files = tokenizer.save_pretrained(model_dir)
155+
tokenizer_path = get_tokenizer_path(model_dir, saved_files)
156+
157+
from executorch.extension.llm.runner import GenerationConfig, TextLLMRunner
158+
159+
runner = TextLLMRunner(f"{model_dir}/model.pte", tokenizer_path)
160+
tokens = []
161+
runner.generate(
162+
"Simply put, the theory of relativity states that",
163+
GenerationConfig(seq_len=64, temperature=0, echo=True),
164+
token_callback=lambda t: tokens.append(t),
161165
)
166+
generated_text = "".join(tokens)
162167
print(f"\nGenerated text:\n\t{generated_text}")
163168
generated_tokens = tokenizer(generated_text, return_tensors="pt").input_ids
164169

165170
# Free memory before loading eager for quality check
166-
del model
171+
del runner
167172
del tokenizer
168173
gc.collect()
169174

‎extension/llm/runner/README.md‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,26 @@ The LLM Runner framework provides Python bindings for easy integration with Pyth
173173
Build the Python bindings as part of the ExecuTorch build:
174174

175175
```bash
176-
# Build from source with Python bindings enabled:
177-
# In executorch root directory
176+
# Option 1: Use the install script (includes pybindings by default)
178177
bash install_executorch.sh
178+
179+
# Option 2: Build with CMake directly
180+
cmake -B cmake-out \
181+
-DEXECUTORCH_BUILD_PYBIND=ON \
182+
-DCMAKE_INSTALL_PREFIX=cmake-out \
183+
cmake/
184+
cmake --build cmake-out -j$(nproc) --target install
185+
186+
# Option 3: pip install from source (includes pybindings)
187+
pip install -e . --no-build-isolation
188+
```
189+
190+
The key CMake flag is `EXECUTORCH_BUILD_PYBIND=ON`, which builds the `_llm_runner` extension module providing `TextLLMRunner`, `MultimodalRunner`, `GenerationConfig`, and related classes.
191+
192+
Verify the installation:
193+
194+
```python
195+
from executorch.extension.llm.runner import TextLLMRunner, GenerationConfig
179196
```
180197

181198
### Quick Start Examples

‎extension/llm/runner/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
MultimodalInput,
2525
MultimodalRunner,
2626
Stats,
27+
TextLLMRunner,
2728
)
2829
except ImportError:
2930
raise RuntimeError(
@@ -231,5 +232,6 @@ def generate_text_hf(
231232
"make_token_input",
232233
"MultimodalInput",
233234
"MultimodalRunner",
235+
"TextLLMRunner",
234236
"Stats",
235237
]

‎extension/llm/runner/_llm_runner.pyi‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,77 @@ class MultimodalInput:
339339

340340
def __repr__(self) -> str: ...
341341

342+
class TextLLMRunner:
343+
"""Runner for text language models."""
344+
345+
def __init__(
346+
self, model_path: str, tokenizer_path: str, data_path: Optional[str] = None
347+
) -> None:
348+
"""
349+
Initialize a TextLLMRunner.
350+
351+
Args:
352+
model_path: Path to the model file (.pte)
353+
tokenizer_path: Path to the tokenizer file
354+
data_path: Optional path to additional data file
355+
Raises:
356+
RuntimeError: If initialization fails
357+
"""
358+
...
359+
360+
def generate(
361+
self,
362+
prompt: str,
363+
config: GenerationConfig,
364+
token_callback: Optional[Callable[[str], None]] = None,
365+
stats_callback: Optional[Callable[[Stats], None]] = None,
366+
) -> None:
367+
"""
368+
Generate text from text input.
369+
370+
Args:
371+
prompt: text input
372+
config: Generation configuration
373+
token_callback: Optional callback called for each generated token
374+
stats_callback: Optional callback called with generation statistics
375+
376+
Raises:
377+
RuntimeError: If generation fails
378+
"""
379+
...
380+
381+
def prefill(self, prompt: str, config: GenerationConfig) -> None:
382+
"""
383+
Prefill text input (e.g., to rebuild KV cache from chat history)
384+
without generating tokens.
385+
386+
Args:
387+
prompt: text input to prefill
388+
389+
Raises:
390+
RuntimeError: If prefill fails
391+
"""
392+
...
393+
394+
def stop(self) -> None:
395+
"""Stop the current generation process."""
396+
...
397+
398+
def reset(self) -> None:
399+
"""Reset the runner state and KV cache."""
400+
...
401+
402+
def get_vocab_size(self) -> int:
403+
"""
404+
Get the vocabulary size of the model.
405+
406+
Returns:
407+
The vocabulary size, or -1 if not available
408+
"""
409+
...
410+
411+
def __repr__(self) -> str: ...
412+
342413
class MultimodalRunner:
343414
"""Runner for multimodal language models."""
344415

‎extension/llm/runner/pybindings.cpp‎

Lines changed: 136 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <executorch/extension/llm/runner/multimodal_input.h>
1818
#include <executorch/extension/llm/runner/multimodal_runner.h>
1919
#include <executorch/extension/llm/runner/stats.h>
20+
#include <executorch/extension/llm/runner/text_llm_runner.h>
2021
#include <executorch/extension/llm/sampler/sampler.h>
2122
#include <executorch/extension/module/module.h>
2223
#include <executorch/runtime/platform/runtime.h>
@@ -42,6 +43,101 @@ using namespace executorch::runtime;
4243
} \
4344
})
4445

46+
// Python wrapper class for TextLLMRunner
47+
class PyTextLLMRunner {
48+
public:
49+
// Constructor that takes a tokenizer path
50+
PyTextLLMRunner(
51+
const std::string& model_path,
52+
const std::string& tokenizer_path,
53+
std::optional<const std::string> data_path = std::nullopt) {
54+
// Load tokenizer using the helper function
55+
auto tokenizer =
56+
load_tokenizer(tokenizer_path, nullptr, std::nullopt, 0, 0);
57+
if (!tokenizer) {
58+
throw std::runtime_error(
59+
"Failed to load tokenizer from: " + tokenizer_path);
60+
}
61+
62+
// Create text llm runner using the helper function
63+
runner_ =
64+
create_text_llm_runner(model_path, std::move(tokenizer), data_path);
65+
if (!runner_) {
66+
throw std::runtime_error(
67+
"Failed to create text llm runner with model: " + model_path);
68+
}
69+
}
70+
71+
void generate(
72+
const std::string& prompt,
73+
const GenerationConfig& config,
74+
py::object token_callback = py::none(),
75+
py::object stats_callback = py::none()) {
76+
if (!runner_) {
77+
throw std::runtime_error("Runner not initialized");
78+
}
79+
80+
// Convert Python callbacks to C++ std::function
81+
std::function<void(const std::string&)> cpp_token_callback = nullptr;
82+
if (!token_callback.is_none()) {
83+
cpp_token_callback = [token_callback](const std::string& token) {
84+
py::gil_scoped_acquire acquire;
85+
token_callback(token);
86+
};
87+
}
88+
89+
std::function<void(const Stats&)> cpp_stats_callback = nullptr;
90+
if (!stats_callback.is_none()) {
91+
cpp_stats_callback = [stats_callback](const Stats& stats) {
92+
py::gil_scoped_acquire acquire;
93+
stats_callback(stats);
94+
};
95+
}
96+
97+
// Release GIL during generation
98+
{
99+
py::gil_scoped_release release;
100+
Error error = runner_->generate(
101+
prompt, config, cpp_token_callback, cpp_stats_callback);
102+
THROW_IF_ERROR(error, "Generation failed");
103+
}
104+
}
105+
106+
void stop() {
107+
if (runner_) {
108+
runner_->stop();
109+
}
110+
}
111+
112+
void reset() {
113+
if (runner_) {
114+
runner_->reset();
115+
}
116+
}
117+
118+
void prefill(const std::string& prompt, const GenerationConfig& config) {
119+
if (!runner_) {
120+
throw std::runtime_error("Runner not initialized");
121+
}
122+
{
123+
py::gil_scoped_release release;
124+
Error error = runner_->prefill(prompt, config);
125+
THROW_IF_ERROR(error, "Prefill failed");
126+
}
127+
}
128+
129+
// Note: Since the runner owns the tokenizer and metadata after creation,
130+
// we cannot directly access them. This is a limitation of the current design.
131+
// For now, we'll return a placeholder value.
132+
int32_t get_vocab_size() const {
133+
// TODO: Consider exposing metadata through the TextLLMRunner interface
134+
return -1; // Indicate that vocab size is not available
135+
}
136+
137+
private:
138+
std::unique_ptr<TextLLMRunner> runner_;
139+
};
140+
45141
// Python wrapper class for MultimodalRunner
46142
class PyMultimodalRunner {
47143
public:
@@ -600,6 +696,45 @@ PYBIND11_MODULE(_llm_runner, m) {
600696
"Create a raw audio input from a torch tensor (batch_size, n_channels, n_samples)",
601697
py::arg("audio_tensor"));
602698

699+
// Bind PyTextLLMRunner
700+
py::class_<PyTextLLMRunner>(m, "TextLLMRunner")
701+
// Constructor with tokenizer path
702+
.def(
703+
py::init<
704+
const std::string&,
705+
const std::string&,
706+
std::optional<const std::string>>(),
707+
py::arg("model_path"),
708+
py::arg("tokenizer_path"),
709+
py::arg("data_path") = py::none(),
710+
"Initialize a TextLLMRunner with model and tokenizer paths")
711+
.def(
712+
"generate",
713+
&PyTextLLMRunner::generate,
714+
py::arg("prompt"),
715+
py::arg("config"),
716+
py::arg("token_callback") = py::none(),
717+
py::arg("stats_callback") = py::none(),
718+
"Generate text from text input with optional callbacks")
719+
.def(
720+
"prefill",
721+
&PyTextLLMRunner::prefill,
722+
py::arg("prompt"),
723+
py::arg("config"),
724+
"Prefill text input (e.g., chat history) without generating tokens")
725+
.def("stop", &PyTextLLMRunner::stop, "Stop the current generation")
726+
.def(
727+
"reset",
728+
&PyTextLLMRunner::reset,
729+
"Reset the runner state and KV cache")
730+
.def(
731+
"get_vocab_size",
732+
&PyTextLLMRunner::get_vocab_size,
733+
"Get the vocabulary size of the model")
734+
.def("__repr__", [](const PyTextLLMRunner& runner) {
735+
return "<TextLLMRunner>";
736+
});
737+
603738
// Bind PyMultimodalRunner
604739
py::class_<PyMultimodalRunner>(m, "MultimodalRunner")
605740
// Constructor with tokenizer path
@@ -644,4 +779,4 @@ PYBIND11_MODULE(_llm_runner, m) {
644779
.def("__repr__", [](const PyMultimodalRunner& runner) {
645780
return "<MultimodalRunner>";
646781
});
647-
}
782+
}

0 commit comments

Comments
 (0)