Skip to content

Commit 381232c

Browse files
authored
support forward and decoder method
Differential Revision: D96962421 Pull Request resolved: pytorch#21746
1 parent 536b985 commit 381232c

5 files changed

Lines changed: 61 additions & 13 deletions

File tree

‎extension/llm/runner/constants.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,11 @@ inline constexpr auto kAudioEncoderMethod = "audio_encoder";
2525
inline constexpr auto kTokenEmbeddingMethod = "token_embedding";
2626
inline constexpr auto kTextModelMethod = "text_decoder";
2727

28+
// Text method name conventions. A single-method PTE exports kForwardMethod; a
29+
// two-method PTE exports kPrefillMethod and kDecodeMethod instead, and is run
30+
// with a dedicated decoder runner per stage.
31+
inline constexpr auto kForwardMethod = "forward";
32+
inline constexpr auto kPrefillMethod = "prefill";
33+
inline constexpr auto kDecodeMethod = "decode";
34+
2835
} // namespace executorch::extension::llm

‎extension/llm/runner/llm_runner_helper.cpp‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -273,15 +273,46 @@ std::unique_ptr<TextLLMRunner> create_text_llm_runner(
273273

274274
auto stats = std::make_unique<Stats>();
275275

276+
auto method_names_result = module->method_names();
277+
const std::unordered_set<std::string> method_names = method_names_result.ok()
278+
? method_names_result.get()
279+
: std::unordered_set<std::string>{};
280+
281+
// A two-method PTE exports kPrefillMethod/kDecodeMethod instead of a single
282+
// forward method, so fall back to kPrefillMethod when the requested method is
283+
// absent. An explicitly requested method that does exist always wins.
284+
std::string prefill_method_name = method_name;
285+
if (method_names.count(prefill_method_name) == 0 &&
286+
method_names.count(kPrefillMethod) > 0) {
287+
prefill_method_name = kPrefillMethod;
288+
}
289+
276290
// Create text_decoder_runner
277-
ET_LOG(Info, "Using method: %s", method_name.c_str());
291+
ET_LOG(Info, "Using method: %s", prefill_method_name.c_str());
278292
auto text_decoder_runner = std::make_unique<TextDecoderRunner>(
279293
module.get(),
280294
io_manager.get(),
281-
method_name,
295+
prefill_method_name,
282296
std::move(sampler),
283297
stats.get());
284298

299+
// The decode stage gets its own decoder runner when the PTE exports a
300+
// separate decode method; otherwise both stages share text_decoder_runner.
301+
std::unique_ptr<TextDecoderRunner> decode_text_decoder_runner;
302+
if (method_names.count(kDecodeMethod) > 0 &&
303+
prefill_method_name != kDecodeMethod) {
304+
ET_LOG(Info, "Using decode method: %s", kDecodeMethod);
305+
decode_text_decoder_runner = std::make_unique<TextDecoderRunner>(
306+
module.get(),
307+
io_manager.get(),
308+
kDecodeMethod,
309+
std::make_unique<Sampler>(vocab_size, init_temp),
310+
stats.get());
311+
}
312+
TextDecoderRunner* decode_runner = decode_text_decoder_runner
313+
? decode_text_decoder_runner.get()
314+
: text_decoder_runner.get();
315+
285316
// Create text_prefiller
286317
auto text_prefiller = std::make_unique<TextPrefiller>(
287318
text_decoder_runner.get(),
@@ -292,7 +323,7 @@ std::unique_ptr<TextLLMRunner> create_text_llm_runner(
292323
// Create text_token_generator with stats
293324
auto text_token_generator = std::make_unique<TextTokenGenerator>(
294325
tokenizer.get(),
295-
text_decoder_runner.get(),
326+
decode_runner,
296327
metadata.at(kUseKVCache),
297328
std::move(eos_ids),
298329
stats.get());
@@ -307,7 +338,8 @@ std::unique_ptr<TextLLMRunner> create_text_llm_runner(
307338
std::move(io_manager),
308339
std::move(text_token_generator),
309340
std::move(stats),
310-
temperature);
341+
temperature,
342+
std::move(decode_text_decoder_runner));
311343
}
312344

313345
std::unique_ptr<MultimodalRunner> create_multimodal_runner(

‎extension/llm/runner/llm_runner_helper.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ ET_EXPERIMENTAL std::unique_ptr<TextLLMRunner> create_text_llm_runner(
108108
std::unique_ptr<::tokenizers::Tokenizer> tokenizer,
109109
std::optional<const std::string> data_path,
110110
float temperature = -1.0f,
111-
const std::string& method_name = "forward",
111+
const std::string& method_name = kForwardMethod,
112112
Module::LoadMode load_mode = Module::LoadMode::MmapUseMlockIgnoreErrors);
113113

114114
/**
@@ -124,7 +124,10 @@ ET_EXPERIMENTAL std::unique_ptr<TextLLMRunner> create_text_llm_runner(
124124
* @param temperature Optional temperature parameter for controlling randomness
125125
* (deprecated)
126126
* @param event_tracer Optional event tracer for profiling
127-
* @param method_name Name of the method to execute in the model
127+
* @param method_name Name of the method to execute in the model. Falls back to
128+
* kPrefillMethod when absent from the model. When the model also exports
129+
* kDecodeMethod, the resolved method drives prefill only and kDecodeMethod
130+
* drives decode.
128131
* @param load_mode Loading strategy for the model file. Defaults to
129132
* MmapUseMlockIgnoreErrors which uses mmap to avoid loading the entire
130133
* model into RAM and attempts to pin pages with mlock for lower inference
@@ -138,7 +141,7 @@ ET_EXPERIMENTAL std::unique_ptr<TextLLMRunner> create_text_llm_runner(
138141
std::vector<std::string> data_files = {},
139142
float temperature = -1.0f,
140143
std::unique_ptr<::executorch::runtime::EventTracer> event_tracer = nullptr,
141-
const std::string& method_name = "forward",
144+
const std::string& method_name = kForwardMethod,
142145
Module::LoadMode load_mode = Module::LoadMode::MmapUseMlockIgnoreErrors);
143146

144147
/**

‎extension/llm/runner/text_llm_runner.cpp‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,20 +35,19 @@ TextLLMRunner::TextLLMRunner(
3535
std::unique_ptr<IOManager> io_manager,
3636
std::unique_ptr<TextTokenGenerator> text_token_generator,
3737
std::unique_ptr<Stats> stats,
38-
float temperature)
38+
float temperature,
39+
std::unique_ptr<TextDecoderRunner> decode_text_decoder_runner)
3940
: tokenizer_(std::move(tokenizer)),
4041
metadata_(std::move(metadata)),
4142
module_(std::move(module)),
4243
text_decoder_runner_(std::move(text_decoder_runner)),
44+
decode_text_decoder_runner_(std::move(decode_text_decoder_runner)),
4345
text_prefiller_(std::move(text_prefiller)),
4446
io_manager_(std::move(io_manager)),
4547
text_token_generator_(std::move(text_token_generator)),
4648
stats_(std::move(stats)),
4749
temperature_(temperature),
48-
pos_(0) {
49-
// Note: This constructor assumes that text_prefiller and text_token_generator
50-
// already have references to the Module and TextDecoderRunner they need
51-
}
50+
pos_(0) {}
5251

5352
bool TextLLMRunner::is_loaded() const {
5453
return text_prefiller_->is_loaded() && text_token_generator_->is_loaded();

‎extension/llm/runner/text_llm_runner.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ class ET_EXPERIMENTAL TextLLMRunner : public IRunner {
5353
* @param stats Statistics tracking object for performance monitoring
5454
* @param temperature Temperature parameter for controlling randomness in
5555
* generation (deprecated). Please use GenerationConfig.temperature instead.
56+
* @param decode_text_decoder_runner Decoder runner for the decode stage of a
57+
* two-method PTE. When null, text_decoder_runner drives both stages.
5658
*/
5759
explicit TextLLMRunner(
5860
std::unordered_map<std::string, int64_t> metadata,
@@ -63,7 +65,8 @@ class ET_EXPERIMENTAL TextLLMRunner : public IRunner {
6365
std::unique_ptr<IOManager> io_manager,
6466
std::unique_ptr<TextTokenGenerator> text_token_generator,
6567
std::unique_ptr<Stats> stats,
66-
float temperature = -1.0f);
68+
float temperature = -1.0f,
69+
std::unique_ptr<TextDecoderRunner> decode_text_decoder_runner = nullptr);
6770

6871
/**
6972
* @brief Checks if the model is loaded and ready for inference
@@ -171,6 +174,10 @@ class ET_EXPERIMENTAL TextLLMRunner : public IRunner {
171174
text_decoder_runner_; // Manage text_decoder_runner_'s lifecycle, make
172175
// sure it outlives text_prefiller_ &
173176
// text_token_generator_.
177+
std::unique_ptr<TextDecoderRunner>
178+
decode_text_decoder_runner_; // Null unless the PTE exports a separate
179+
// decode method, in which case
180+
// text_decoder_runner_ drives prefill only.
174181
std::unique_ptr<TextPrefiller> text_prefiller_;
175182
std::unique_ptr<IOManager> io_manager_;
176183
std::unique_ptr<TextTokenGenerator> text_token_generator_;

0 commit comments

Comments
 (0)