Skip to content

Commit a6bfb3e

Browse files
authored
Allow TextDecoderRunner to reuse a persistent Sampler (pytorch#19351)
Differential Revision: D103748189 Pull Request resolved: pytorch#19351
1 parent 51b06fd commit a6bfb3e

8 files changed

Lines changed: 153 additions & 30 deletions

File tree

‎extension/llm/runner/llm_runner_helper.cpp‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -266,10 +266,15 @@ std::unique_ptr<TextLLMRunner> create_text_llm_runner(
266266
// Create IOManager
267267
std::unique_ptr<IOManager> io_manager = std::make_unique<IOManager>(*module);
268268

269+
// Read vocab_size for Sampler
270+
int32_t vocab_size = static_cast<int32_t>(metadata.at(kVocabSize));
271+
float init_temp = temperature == -1.0f ? 0.0f : temperature;
272+
auto sampler = std::make_unique<Sampler>(vocab_size, init_temp);
273+
269274
// Create text_decoder_runner
270275
ET_LOG(Info, "Using method: %s", method_name.c_str());
271276
auto text_decoder_runner = std::make_unique<TextDecoderRunner>(
272-
module.get(), io_manager.get(), method_name);
277+
module.get(), io_manager.get(), method_name, std::move(sampler));
273278

274279
// Create text_prefiller
275280
auto text_prefiller = std::make_unique<TextPrefiller>(
@@ -334,9 +339,13 @@ std::unique_ptr<MultimodalRunner> create_multimodal_runner(
334339
// Create IOManager
335340
std::unique_ptr<IOManager> io_manager = std::make_unique<IOManager>(*module);
336341

342+
// Read vocab_size for Sampler
343+
int32_t vocab_size = static_cast<int32_t>(metadata.at(kVocabSize));
344+
auto sampler = std::make_unique<Sampler>(vocab_size, 0.0f); // Default temp
345+
337346
// Create text_decoder_runner
338-
auto text_decoder_runner =
339-
std::make_unique<MultimodalDecoderRunner>(module.get(), io_manager.get());
347+
auto text_decoder_runner = std::make_unique<MultimodalDecoderRunner>(
348+
module.get(), io_manager.get(), "forward", std::move(sampler));
340349

341350
// Create multimodal_prefiller
342351
auto multimodal_prefiller = std::make_unique<MultimodalPrefiller>(

‎extension/llm/runner/multimodal_decoder_runner.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,16 @@ namespace executorch::extension::llm {
1515
class ET_EXPERIMENTAL MultimodalDecoderRunner
1616
: public executorch::extension::llm::TextDecoderRunner {
1717
public:
18-
explicit MultimodalDecoderRunner(Module* module, IOManager* io_manager)
19-
: TextDecoderRunner(module, io_manager) {}
18+
explicit MultimodalDecoderRunner(
19+
Module* module,
20+
IOManager* io_manager,
21+
std::string method_name = "forward",
22+
std::unique_ptr<Sampler> sampler = nullptr)
23+
: TextDecoderRunner(
24+
module,
25+
io_manager,
26+
std::move(method_name),
27+
std::move(sampler)) {}
2028

2129
/**
2230
* Step the LLM Decoder with the given tokens and start position.

‎extension/llm/runner/test/test_text_decoder_runner.cpp‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,83 @@ TEST_F(TextDecoderRunnerTest, LogitsToTokenWithTemperature) {
155155
EXPECT_LT(token, 4);
156156
}
157157

158+
// Test logits_to_token() with an injected Sampler (greedy, temp=0)
159+
TEST_F(TextDecoderRunnerTest, LogitsToTokenWithInjectedSampler) {
160+
TensorFactory<executorch::aten::ScalarType::Float> tf_float;
161+
auto logits = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
162+
163+
auto sampler = std::make_unique<executorch::extension::llm::Sampler>(4, 0.0f);
164+
auto runner = std::make_unique<TextDecoderRunner>(
165+
nullptr, nullptr, "forward", std::move(sampler));
166+
167+
int32_t token = runner->logits_to_token(logits, 0.0f);
168+
EXPECT_EQ(token, 2);
169+
170+
auto logits2 = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
171+
token = runner->logits_to_token(logits2, 0.0f);
172+
EXPECT_EQ(token, 2);
173+
}
174+
175+
// Test that set_temperature works on an injected Sampler
176+
TEST_F(TextDecoderRunnerTest, LogitsToTokenInjectedSamplerTemperatureSwitch) {
177+
auto sampler = std::make_unique<executorch::extension::llm::Sampler>(4, 0.0f);
178+
auto runner = std::make_unique<TextDecoderRunner>(
179+
nullptr, nullptr, "forward", std::move(sampler));
180+
181+
TensorFactory<executorch::aten::ScalarType::Float> tf_float;
182+
183+
// temp=0 → argmax
184+
auto logits1 = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
185+
EXPECT_EQ(runner->logits_to_token(logits1, 0.0f), 2);
186+
187+
// temp=1.0 → stochastic, result must be in valid range
188+
auto logits2 = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
189+
int32_t token = runner->logits_to_token(logits2, 1.0f);
190+
EXPECT_GE(token, 0);
191+
EXPECT_LT(token, 4);
192+
193+
// temp=0 again → back to argmax
194+
auto logits3 = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
195+
EXPECT_EQ(runner->logits_to_token(logits3, 0.0f), 2);
196+
}
197+
198+
// Test logits_to_token() with an injected Sampler on a 3D tensor
199+
TEST_F(TextDecoderRunnerTest, LogitsToTokenWithInjectedSampler3D) {
200+
TensorFactory<executorch::aten::ScalarType::Float> tf_float;
201+
auto logits = tf_float.make(
202+
{1, 2, 4},
203+
{
204+
0.1f,
205+
0.2f,
206+
0.3f,
207+
0.4f, // first position
208+
0.5f,
209+
0.6f,
210+
0.9f,
211+
0.8f // last position (used for sampling)
212+
});
213+
214+
auto sampler = std::make_unique<executorch::extension::llm::Sampler>(4, 0.0f);
215+
auto runner = std::make_unique<TextDecoderRunner>(
216+
nullptr, nullptr, "forward", std::move(sampler));
217+
218+
int32_t token = runner->logits_to_token(logits, 0.0f);
219+
EXPECT_EQ(token, 2);
220+
}
221+
222+
// Test logits_to_token() without an injected Sampler (fallback path)
223+
TEST_F(TextDecoderRunnerTest, LogitsToTokenWithoutInjectedSampler) {
224+
TensorFactory<executorch::aten::ScalarType::Float> tf_float;
225+
auto logits = tf_float.make({1, 4}, {0.1f, 0.2f, 0.8f, 0.4f});
226+
227+
// No sampler injected — uses the legacy per-call Sampler path
228+
auto runner =
229+
std::make_unique<TextDecoderRunner>(nullptr, nullptr, "forward");
230+
231+
int32_t token = runner->logits_to_token(logits, 0.0f);
232+
EXPECT_EQ(token, 2); // greedy argmax → index 2 (value 0.8)
233+
}
234+
158235
// Test step() method with all available PTE models
159236
TEST_F(TextDecoderRunnerTest, StepWithAllModels) {
160237
// List of all environment variables for PTE models

‎extension/llm/runner/text_decoder_runner.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@ namespace llm {
2525
TextDecoderRunner::TextDecoderRunner(
2626
Module* module,
2727
IOManager* io_manager,
28-
std::string method_name)
28+
std::string method_name,
29+
std::unique_ptr<Sampler> sampler)
2930
: module_(module),
3031
io_manager_(io_manager),
31-
method_name_(std::move(method_name)) {}
32+
method_name_(std::move(method_name)),
33+
sampler_(std::move(sampler)) {}
3234

3335
// This function is functional, meaning it shouldn't modify any state of the
3436
// input. It should be safe to call multiple times with the same inputs. The

‎extension/llm/runner/text_decoder_runner.h‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ class ET_EXPERIMENTAL TextDecoderRunner {
2323
explicit TextDecoderRunner(
2424
Module* module,
2525
IOManager* io_manager,
26-
std::string method_name = "forward");
26+
std::string method_name = "forward",
27+
std::unique_ptr<Sampler> sampler = nullptr);
2728

2829
virtual ~TextDecoderRunner() = default;
2930

@@ -71,14 +72,20 @@ class ET_EXPERIMENTAL TextDecoderRunner {
7172

7273
/**
7374
* Sample the next token from the logits tensor.
74-
* @param logits_tensor The logits tensor.
75-
* @param temperature The temperature parameter used to control randomness in
76-
* sampling.
77-
* @return The next token.
75+
* If a Sampler was passed in the constructor, it is reused (its temperature
76+
* will be updated to match the argument). Otherwise a temporary Sampler is
77+
* created per call.
78+
* @note Not thread-safe when a Sampler is injected; callers must serialize
79+
* access to this method.
7880
*/
7981
inline int32_t logits_to_token(
8082
const executorch::aten::Tensor& logits_tensor,
8183
const float temperature = 0.0f) {
84+
if (sampler_) {
85+
sampler_->set_temperature(temperature);
86+
return ::executorch::extension::llm::sample_from_logits(
87+
logits_tensor, *sampler_);
88+
}
8289
return ::executorch::extension::llm::logits_to_token(
8390
logits_tensor, temperature);
8491
}
@@ -94,6 +101,7 @@ class ET_EXPERIMENTAL TextDecoderRunner {
94101
Module* module_;
95102
IOManager* io_manager_;
96103
std::string method_name_;
104+
std::unique_ptr<Sampler> sampler_;
97105
};
98106

99107
} // namespace llm

‎extension/llm/runner/text_llm_runner.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,14 @@ Error TextLLMRunner::generate(
235235
// Set ignore_eos based on config
236236
text_token_generator_->set_ignore_eos(config.ignore_eos);
237237

238+
// Use the configuration's temperature
239+
float resolved_temp =
240+
temperature_ == -1.0f ? config.temperature : temperature_;
241+
238242
// Generate max_new_tokens - 1 because prefill already generated 1 token.
239243
auto generate_result = text_token_generator_->generate(
240-
prompt_tokens,
241-
pos_,
242-
max_new_tokens - 1,
243-
temperature_ == -1.0f ? config.temperature : temperature_,
244-
wrapped_callback);
244+
prompt_tokens, pos_, max_new_tokens - 1, resolved_temp, wrapped_callback);
245+
245246
if (!generate_result.ok()) {
246247
return generate_result.error();
247248
}

‎extension/llm/sampler/sampler.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@ class ET_EXPERIMENTAL Sampler {
4444

4545
Sampler(int32_t vocab_size, float temperature);
4646

47+
/// Update the sampling temperature. Invalid (< 0) values are reset to 0.
48+
void set_temperature(float temperature) {
49+
if (temperature < 0.0f) {
50+
temperature = 0.0f;
51+
}
52+
inv_temperature_ = (temperature != 0.0f) ? 1.0f / temperature : 0.0f;
53+
}
54+
4755
// Enable top-k filtering. k <= 0 or k >= vocab_size disables top-k.
4856
// When top-k is enabled, top-p is ignored — the two modes are mutually
4957
// exclusive in this implementation.

‎extension/llm/sampler/util.h‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,20 @@ namespace extension {
1616
namespace llm {
1717

1818
/**
19-
* Sample the next token from the logits tensor.
19+
* Sample the next token from the logits tensor using a pre-configured Sampler.
2020
* @param logits_tensor The logits tensor.
21-
* @param temperature The temperature parameter used to control randomness in
22-
* sampling.
21+
* @param sampler The sampler to use for token selection.
2322
* @return The next token.
23+
* @note The sampler's vocab_size must match logits_tensor's last dimension.
2424
*/
25-
inline int32_t logits_to_token(
25+
inline int32_t sample_from_logits(
2626
const executorch::aten::Tensor& logits_tensor,
27-
const float temperature = 0.0f) {
27+
Sampler& sampler) {
2828
int32_t result = 0;
2929

30-
// Create a minimal context for error handling in ET_SWITCH
3130
struct {
3231
[[noreturn]] void fail(torch::executor::Error /* error */) {
33-
ET_CHECK_MSG(false, "Unsupported dtype in logits_to_token");
32+
ET_CHECK_MSG(false, "Unsupported dtype in sample_from_logits");
3433
}
3534
} ctx;
3635

@@ -41,25 +40,36 @@ inline int32_t logits_to_token(
4140
UInt16,
4241
logits_tensor.scalar_type(),
4342
ctx,
44-
"logits_to_token",
43+
"sample_from_logits",
4544
CTYPE,
4645
[&]() {
47-
// If the logit_tensor rank is 3, the shape is [batch, seq_length,
48-
// vocab_size], get the last logits, sample and return. Else the model
49-
// outputs the last logit, directly sample and return.
5046
auto* logits = logits_tensor.mutable_data_ptr<CTYPE>();
5147
ssize_t vocab_size = logits_tensor.size(logits_tensor.dim() - 1);
5248
if (logits_tensor.dim() == 3) {
5349
auto num_tokens = logits_tensor.size(1);
5450
logits += (num_tokens - 1) * vocab_size;
5551
}
56-
// @lint-ignore CLANGTIDY facebook-hte-Deprecated
57-
Sampler sampler(vocab_size, temperature);
5852
result = sampler.sample(logits);
5953
});
6054
return result;
6155
}
6256

57+
/**
58+
* Sample the next token from the logits tensor.
59+
* @param logits_tensor The logits tensor.
60+
* @param temperature The temperature parameter used to control randomness in
61+
* sampling.
62+
* @return The next token.
63+
*/
64+
inline int32_t logits_to_token(
65+
const executorch::aten::Tensor& logits_tensor,
66+
const float temperature = 0.0f) {
67+
ssize_t vocab_size = logits_tensor.size(logits_tensor.dim() - 1);
68+
// @lint-ignore CLANGTIDY facebook-hte-Deprecated
69+
Sampler sampler(vocab_size, temperature);
70+
return sample_from_logits(logits_tensor, sampler);
71+
}
72+
6373
} // namespace llm
6474
} // namespace extension
6575
} // namespace executorch

0 commit comments

Comments
 (0)