1717#include < executorch/extension/llm/runner/multimodal_input.h>
1818#include < executorch/extension/llm/runner/multimodal_runner.h>
1919#include < executorch/extension/llm/runner/stats.h>
20+ #include < executorch/extension/llm/runner/text_llm_runner.h>
2021#include < executorch/extension/llm/sampler/sampler.h>
2122#include < executorch/extension/module/module.h>
2223#include < executorch/runtime/platform/runtime.h>
@@ -42,6 +43,101 @@ using namespace executorch::runtime;
4243 } \
4344 })
4445
46+ // Python wrapper class for TextLLMRunner
47+ class PyTextLLMRunner {
48+ public:
49+ // Constructor that takes a tokenizer path
50+ PyTextLLMRunner (
51+ const std::string& model_path,
52+ const std::string& tokenizer_path,
53+ std::optional<const std::string> data_path = std::nullopt ) {
54+ // Load tokenizer using the helper function
55+ auto tokenizer =
56+ load_tokenizer (tokenizer_path, nullptr , std::nullopt , 0 , 0 );
57+ if (!tokenizer) {
58+ throw std::runtime_error (
59+ " Failed to load tokenizer from: " + tokenizer_path);
60+ }
61+
62+ // Create text llm runner using the helper function
63+ runner_ =
64+ create_text_llm_runner (model_path, std::move (tokenizer), data_path);
65+ if (!runner_) {
66+ throw std::runtime_error (
67+ " Failed to create text llm runner with model: " + model_path);
68+ }
69+ }
70+
71+ void generate (
72+ const std::string& prompt,
73+ const GenerationConfig& config,
74+ py::object token_callback = py::none(),
75+ py::object stats_callback = py::none()) {
76+ if (!runner_) {
77+ throw std::runtime_error (" Runner not initialized" );
78+ }
79+
80+ // Convert Python callbacks to C++ std::function
81+ std::function<void (const std::string&)> cpp_token_callback = nullptr ;
82+ if (!token_callback.is_none ()) {
83+ cpp_token_callback = [token_callback](const std::string& token) {
84+ py::gil_scoped_acquire acquire;
85+ token_callback (token);
86+ };
87+ }
88+
89+ std::function<void (const Stats&)> cpp_stats_callback = nullptr ;
90+ if (!stats_callback.is_none ()) {
91+ cpp_stats_callback = [stats_callback](const Stats& stats) {
92+ py::gil_scoped_acquire acquire;
93+ stats_callback (stats);
94+ };
95+ }
96+
97+ // Release GIL during generation
98+ {
99+ py::gil_scoped_release release;
100+ Error error = runner_->generate (
101+ prompt, config, cpp_token_callback, cpp_stats_callback);
102+ THROW_IF_ERROR (error, " Generation failed" );
103+ }
104+ }
105+
106+ void stop () {
107+ if (runner_) {
108+ runner_->stop ();
109+ }
110+ }
111+
112+ void reset () {
113+ if (runner_) {
114+ runner_->reset ();
115+ }
116+ }
117+
118+ void prefill (const std::string& prompt, const GenerationConfig& config) {
119+ if (!runner_) {
120+ throw std::runtime_error (" Runner not initialized" );
121+ }
122+ {
123+ py::gil_scoped_release release;
124+ Error error = runner_->prefill (prompt, config);
125+ THROW_IF_ERROR (error, " Prefill failed" );
126+ }
127+ }
128+
129+ // Note: Since the runner owns the tokenizer and metadata after creation,
130+ // we cannot directly access them. This is a limitation of the current design.
131+ // For now, we'll return a placeholder value.
132+ int32_t get_vocab_size () const {
133+ // TODO: Consider exposing metadata through the TextLLMRunner interface
134+ return -1 ; // Indicate that vocab size is not available
135+ }
136+
137+ private:
138+ std::unique_ptr<TextLLMRunner> runner_;
139+ };
140+
45141// Python wrapper class for MultimodalRunner
46142class PyMultimodalRunner {
47143 public:
@@ -600,6 +696,45 @@ PYBIND11_MODULE(_llm_runner, m) {
600696 " Create a raw audio input from a torch tensor (batch_size, n_channels, n_samples)" ,
601697 py::arg (" audio_tensor" ));
602698
699+ // Bind PyTextLLMRunner
700+ py::class_<PyTextLLMRunner>(m, " TextLLMRunner" )
701+ // Constructor with tokenizer path
702+ .def (
703+ py::init<
704+ const std::string&,
705+ const std::string&,
706+ std::optional<const std::string>>(),
707+ py::arg (" model_path" ),
708+ py::arg (" tokenizer_path" ),
709+ py::arg (" data_path" ) = py::none (),
710+ " Initialize a TextLLMRunner with model and tokenizer paths" )
711+ .def (
712+ " generate" ,
713+ &PyTextLLMRunner::generate,
714+ py::arg (" prompt" ),
715+ py::arg (" config" ),
716+ py::arg (" token_callback" ) = py::none (),
717+ py::arg (" stats_callback" ) = py::none (),
718+ " Generate text from text input with optional callbacks" )
719+ .def (
720+ " prefill" ,
721+ &PyTextLLMRunner::prefill,
722+ py::arg (" prompt" ),
723+ py::arg (" config" ),
724+ " Prefill text input (e.g., chat history) without generating tokens" )
725+ .def (" stop" , &PyTextLLMRunner::stop, " Stop the current generation" )
726+ .def (
727+ " reset" ,
728+ &PyTextLLMRunner::reset,
729+ " Reset the runner state and KV cache" )
730+ .def (
731+ " get_vocab_size" ,
732+ &PyTextLLMRunner::get_vocab_size,
733+ " Get the vocabulary size of the model" )
734+ .def (" __repr__" , [](const PyTextLLMRunner& runner) {
735+ return " <TextLLMRunner>" ;
736+ });
737+
603738 // Bind PyMultimodalRunner
604739 py::class_<PyMultimodalRunner>(m, " MultimodalRunner" )
605740 // Constructor with tokenizer path
@@ -644,4 +779,4 @@ PYBIND11_MODULE(_llm_runner, m) {
644779 .def (" __repr__" , [](const PyMultimodalRunner& runner) {
645780 return " <MultimodalRunner>" ;
646781 });
647- }
782+ }
0 commit comments