Skip to content

Commit ec22b0d

Browse files
committed
code cleanup
1 parent 6c0167e commit ec22b0d

7 files changed

Lines changed: 33 additions & 324 deletions

File tree

‎packages/react-native-executorch/common/runner/multimodal_decoder_runner.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,24 @@ class MultimodalDecoderRunner : public TextDecoderRunner {
8080
return outputs[0].toTensor();
8181
}
8282

83+
inline ::executorch::runtime::Result<::executorch::aten::Tensor>
84+
decode(const ::executorch::runtime::EValue &embeddings,
85+
const ::executorch::runtime::EValue &ple_tok, int64_t start_pos) {
86+
auto start_pos_tensor = ::executorch::extension::from_blob(
87+
&start_pos, {1}, ::executorch::aten::ScalarType::Long);
88+
auto outputs_result = module_->execute(
89+
kTextModelMethod, {embeddings, ple_tok, start_pos_tensor});
90+
if (!outputs_result.ok()) {
91+
return outputs_result.error();
92+
}
93+
auto &outputs = *outputs_result;
94+
ET_CHECK_MSG(outputs.size() == 1,
95+
"Expected 1 output from text_decoder, got %zu",
96+
outputs.size());
97+
ET_CHECK_MSG(outputs[0].isTensor(), "text_decoder output is not a tensor");
98+
return outputs[0].toTensor();
99+
}
100+
83101
inline ::executorch::runtime::Error load() override {
84102
if (is_method_loaded()) {
85103
return ::executorch::runtime::Error::Ok;

‎packages/react-native-executorch/common/runner/multimodal_prefiller.cpp‎

Lines changed: 4 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
#include "constants.h"
2121
#include "util.h"
2222
#include <algorithm>
23-
#include <cmath>
24-
#include <cstdio>
2523
#include <cstring>
2624
#include <rnexecutorch/Log.h>
2725
#include <string>
@@ -44,8 +42,6 @@ MultimodalPrefiller::MultimodalPrefiller(
4442
Result<uint64_t>
4543
MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
4644
int64_t &start_pos) {
47-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa multimodal prefill");
48-
4945
const bool has_ple = decoder_runner_->has_ple();
5046
const long t_prefill_begin = time_in_ms();
5147

@@ -138,11 +134,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
138134
int audio_calls = 0;
139135

140136
for (const auto &input : inputs) {
141-
142-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa input type",
143-
input.is_audio() ? "audio" : "no audio",
144-
input.is_image() ? "image" : "no image");
145-
146137
if (input.is_image()) {
147138
ET_CHECK_OR_RETURN_ERROR(image_encoder_ != nullptr, InvalidState,
148139
"No image encoder registered");
@@ -156,9 +147,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
156147
ET_CHECK_OR_RETURN_ERROR(audio_encoder_ != nullptr, InvalidState,
157148
"No audio encoder registered");
158149
const long t_aud_begin = time_in_ms();
159-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa encoding audio");
160150
auto enc = audio_encoder_->encode(input);
161-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa encoded audio");
162151
ET_CHECK_OK_OR_RETURN_ERROR(enc.error(), "Audio encoding failed");
163152
audio_encode_ms += time_in_ms() - t_aud_begin;
164153
audio_calls += 1;
@@ -173,8 +162,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
173162
audio_tensor.dim());
174163
const int64_t num_audio = static_cast<int64_t>(audio_tensor.size(1));
175164
const int64_t audio_hidden = static_cast<int64_t>(audio_tensor.size(2));
176-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa encoded audio",
177-
num_audio, audio_hidden);
178165
ET_CHECK_OR_RETURN_ERROR(num_audio > 0, InvalidState,
179166
"Audio encoder produced 0 tokens");
180167
std::vector<uint8_t> bytes(audio_tensor.nbytes());
@@ -184,62 +171,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
184171
AudioSlot{std::move(bytes), audio_tensor.scalar_type(),
185172
static_cast<int64_t>(ids.size()), num_audio, audio_hidden});
186173
ids.insert(ids.end(), static_cast<size_t>(num_audio), 0);
187-
188-
// Diagnostic: dump audio encoder output magnitude + first values so
189-
// the user can see in the JS console whether the encoder is producing
190-
// real embeddings or zeros/noise. slot.bytes can be either fp32 or
191-
// fp16 depending on exporter; handle both.
192-
{
193-
const AudioSlot &slot = audio_slots.back();
194-
const size_t nrows = static_cast<size_t>(slot.num_audio);
195-
const size_t hidden = static_cast<size_t>(slot.audio_hidden);
196-
const size_t nfloats = nrows * hidden;
197-
const size_t bytes_per_elem =
198-
nfloats > 0 ? slot.bytes.size() / nfloats : 0;
199-
float maxabs = 0.0f;
200-
double sumsq = 0.0;
201-
std::string head;
202-
const size_t headN = std::min<size_t>(16, nfloats);
203-
if (slot.dtype == ::executorch::aten::ScalarType::Float) {
204-
const float *e = reinterpret_cast<const float *>(slot.bytes.data());
205-
for (size_t i = 0; i < nfloats; ++i) {
206-
const float a = std::fabs(e[i]);
207-
if (a > maxabs)
208-
maxabs = a;
209-
sumsq += static_cast<double>(e[i]) * static_cast<double>(e[i]);
210-
}
211-
for (size_t i = 0; i < headN; ++i) {
212-
if (i)
213-
head += ", ";
214-
head += std::to_string(e[i]);
215-
}
216-
} else if (slot.dtype == ::executorch::aten::ScalarType::Half) {
217-
const auto *e = reinterpret_cast<const ::executorch::aten::Half *>(
218-
slot.bytes.data());
219-
for (size_t i = 0; i < nfloats; ++i) {
220-
const float v = static_cast<float>(e[i]);
221-
const float a = std::fabs(v);
222-
if (a > maxabs)
223-
maxabs = a;
224-
sumsq += static_cast<double>(v) * static_cast<double>(v);
225-
}
226-
for (size_t i = 0; i < headN; ++i) {
227-
if (i)
228-
head += ", ";
229-
head += std::to_string(static_cast<float>(e[i]));
230-
}
231-
} else {
232-
head = "<unsupported dtype>";
233-
}
234-
const double rms =
235-
nfloats > 0 ? std::sqrt(sumsq / static_cast<double>(nfloats)) : 0.0;
236-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info,
237-
"kappa [AudioEmbed] num_audio=", nrows,
238-
" hidden=", hidden, " slot_start=", slot.slot_start,
239-
" dtype=", static_cast<int>(slot.dtype),
240-
" bytes_per_elem=", bytes_per_elem,
241-
" maxabs=", maxabs, " rms=", rms, " first16=", head);
242-
}
243174
} else if (input.is_text() || input.is_tokens()) {
244175
std::vector<uint64_t> tokens;
245176
if (input.is_text()) {
@@ -279,42 +210,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
279210
ids.resize(static_cast<size_t>(max_seq_len), 0);
280211
}
281212

282-
// ------------------------------------------------------------
283-
// Splice diagnostics: dump the fused id stream + slot map so the JS
284-
// console can confirm placeholder positions (audio/image use id=0)
285-
// line up with where the encoder outputs will be spliced in pass 2/2b.
286-
// ------------------------------------------------------------
287-
{
288-
std::string ids_dump;
289-
const size_t dump_n = std::min<size_t>(ids.size(), size_t{512});
290-
ids_dump.reserve(dump_n * 6);
291-
for (size_t i = 0; i < dump_n; ++i) {
292-
if (i)
293-
ids_dump += ",";
294-
ids_dump += std::to_string(ids[i]);
295-
}
296-
if (ids.size() > dump_n)
297-
ids_dump += ",…";
298-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info,
299-
"[Splice] total_len=", total_len, " ids_buf=", ids.size(),
300-
" image_slots=", image_slots.size(),
301-
" audio_slots=", audio_slots.size(), " ids[0..", dump_n,
302-
"]=", ids_dump);
303-
for (size_t i = 0; i < image_slots.size(); ++i) {
304-
const auto &s = image_slots[i];
305-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "[Splice] image_slot[",
306-
i, "] start=", s.slot_start, " num=", s.num_visual);
307-
}
308-
for (size_t i = 0; i < audio_slots.size(); ++i) {
309-
const auto &s = audio_slots[i];
310-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "[Splice] audio_slot[",
311-
i, "] start=", s.slot_start, " num=", s.num_audio,
312-
" hidden=", s.audio_hidden,
313-
" dtype=", static_cast<int>(s.dtype),
314-
" bytes=", s.bytes.size());
315-
}
316-
}
317-
318213
// ------------------------------------------------------------
319214
// Single token_embedding call over the fused id buffer.
320215
// ------------------------------------------------------------
@@ -393,7 +288,10 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
393288
const long t_vis_begin = time_in_ms();
394289
auto encode_result = image_encoder_->encode(*slot.input);
395290
ET_CHECK_OK_OR_RETURN_ERROR(encode_result.error(), "Image encoding failed");
396-
encoder_output = *encode_result;
291+
vision_total_ms += time_in_ms() - t_vis_begin;
292+
vision_calls += 1;
293+
auto encoder_output = *encode_result;
294+
auto vision_tensor = encoder_output.toTensor();
397295

398296
const auto vision_dtype = vision_tensor.scalar_type();
399297
const size_t visual_elems =
@@ -433,44 +331,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
433331
// byte snapshot taken at encode time so post-encode execute() calls can't
434332
// invalidate slot state. Same dtype-conversion matrix as vision.
435333
// ------------------------------------------------------------
436-
437-
// Diagnostic helper: stringify the first N elements of a byte buffer
438-
// interpreted as the given scalar dtype (Float or Half supported). Used
439-
// for the before/src/after snapshots logged per audio slot so the user
440-
// can verify (a) the destination held pad-token embeds prior to splice,
441-
// (b) the source bytes carry real audio encoder output, and (c) the
442-
// destination contains the audio values post-splice.
443-
auto stringify_floats = [](const uint8_t *p, size_t n,
444-
::executorch::aten::ScalarType dt) -> std::string {
445-
std::string out;
446-
out.reserve(n * 10);
447-
for (size_t i = 0; i < n; ++i) {
448-
if (i)
449-
out += ",";
450-
float v = 0.0f;
451-
if (dt == ::executorch::aten::ScalarType::Float) {
452-
v = reinterpret_cast<const float *>(p)[i];
453-
} else if (dt == ::executorch::aten::ScalarType::Half) {
454-
v = static_cast<float>(
455-
reinterpret_cast<const ::executorch::aten::Half *>(p)[i]);
456-
}
457-
char buf[24];
458-
std::snprintf(buf, sizeof(buf), "%.4f", v);
459-
out += buf;
460-
}
461-
return out;
462-
};
463-
464-
rnexecutorch::log(
465-
rnexecutorch::LOG_LEVEL::Info,
466-
"kappa [Splice] pass 2b begin: audio_slots=", audio_slots.size(),
467-
" embeds_buf_bytes=", embeds_buf.size(), " hidden=", hidden,
468-
" embeds_dtype=", static_cast<int>(embeds_dtype),
469-
" embeds_elem_size=", embeds_elem_size);
470-
471334
for (auto &slot : audio_slots) {
472-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa splice loop!!!");
473-
474335
ET_CHECK_OR_RETURN_ERROR(
475336
slot.audio_hidden == static_cast<int64_t>(hidden), InvalidState,
476337
"audio encoder hidden %lld != text_embed hidden %lld",
@@ -494,16 +355,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
494355
static_cast<size_t>(hidden) *
495356
embeds_elem_size;
496357

497-
// Snapshot the first row before splice. These bytes should hold the
498-
// pad-token (id=0) embedding because ids.insert pushed zeros into
499-
// these slot positions before the token_embedding call. If `before`
500-
// already equals `src` you are looking at a re-run; if `after` differs
501-
// from `src` the splice/dtype-cast path is wrong.
502-
const size_t dbg_n = std::min<size_t>(8, static_cast<size_t>(hidden));
503-
const std::string before_str = stringify_floats(dst, dbg_n, embeds_dtype);
504-
const std::string src_str =
505-
stringify_floats(slot.bytes.data(), dbg_n, audio_dtype);
506-
507358
if (audio_dtype == embeds_dtype) {
508359
std::memcpy(dst, slot.bytes.data(), audio_elems * embeds_elem_size);
509360
} else if (audio_dtype == ::executorch::aten::ScalarType::Float &&
@@ -527,40 +378,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
527378
"unsupported audio/text dtype pair: audio=%hhd text=%hhd",
528379
static_cast<int8_t>(audio_dtype), static_cast<int8_t>(embeds_dtype));
529380
}
530-
531-
// Post-splice: re-read the same dst row + compute magnitude over the
532-
// entire spliced region so the user can confirm the bytes flipped to
533-
// real audio embeds (post should equal src, post_maxabs > 0 always).
534-
const std::string after_str = stringify_floats(dst, dbg_n, embeds_dtype);
535-
float maxabs = 0.0f;
536-
double sumsq = 0.0;
537-
for (size_t i = 0; i < audio_elems; ++i) {
538-
float v = 0.0f;
539-
if (embeds_dtype == ::executorch::aten::ScalarType::Float) {
540-
v = reinterpret_cast<const float *>(dst)[i];
541-
} else if (embeds_dtype == ::executorch::aten::ScalarType::Half) {
542-
v = static_cast<float>(
543-
reinterpret_cast<const ::executorch::aten::Half *>(dst)[i]);
544-
}
545-
const float av = std::fabs(v);
546-
if (av > maxabs)
547-
maxabs = av;
548-
sumsq += static_cast<double>(v) * static_cast<double>(v);
549-
}
550-
const double rms = audio_elems > 0
551-
? std::sqrt(sumsq / static_cast<double>(audio_elems))
552-
: 0.0;
553-
554-
rnexecutorch::log(
555-
rnexecutorch::LOG_LEVEL::Info,
556-
"kappa [Splice] audio slot start=", slot.slot_start,
557-
" num=", slot.num_audio, " dtype src=", static_cast<int>(audio_dtype),
558-
" dst=", static_cast<int>(embeds_dtype), " pre[0..", dbg_n,
559-
"]=", before_str, " src[0..", dbg_n, "]=", src_str, " post[0..", dbg_n,
560-
"]=", after_str, " post_maxabs=", maxabs, " post_rms=", rms);
561381
}
562-
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa post splice",
563-
embeds_buf);
564382

565383
// ------------------------------------------------------------
566384
// Chunked text_decoder calls.

‎packages/react-native-executorch/common/runner/text_decoder_runner.cpp‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,7 @@ TextDecoderRunner::TextDecoderRunner(Module &module, IOManager *io_manager,
3131
// outer loop (call site) is responsible for managing state.
3232
::executorch::runtime::Result<executorch::aten::Tensor>
3333
TextDecoderRunner::step(TensorPtr &tokens, int64_t start_pos) {
34-
// Pick method by phase: parallel-prefill chunks (>1 token) go through the
35-
// dynamic "forward"; single-token decode steps go through the static
36-
// "forward_decode" method when present (avoids dynamic-shape re-encode
37-
// overhead — measured 8× faster on gemma4 Vulkan).
38-
const char *method_name = "forward";
39-
auto method_meta_result = module_->method_meta(method_name);
34+
auto method_meta_result = module_->method_meta("forward");
4035
if (!method_meta_result.ok()) {
4136
return method_meta_result.error();
4237
}
@@ -48,7 +43,7 @@ TextDecoderRunner::step(TensorPtr &tokens, int64_t start_pos) {
4843

4944
if (use_kv_cache) {
5045
auto start_pos_tensor_result = populate_start_pos_or_cache_position(
51-
module_, start_pos, cache_positions, tokens->numel(), method_name);
46+
module_, start_pos, cache_positions, tokens->numel(), "forward");
5247
if (!start_pos_tensor_result.ok()) {
5348
return start_pos_tensor_result.error();
5449
}
@@ -58,7 +53,7 @@ TextDecoderRunner::step(TensorPtr &tokens, int64_t start_pos) {
5853
auto inputs_res = io_manager_->prepare_decode(tokens, start_pos_tensor);
5954
ET_CHECK_OK_OR_RETURN_ERROR(inputs_res.error());
6055
inputs = inputs_res.get();
61-
auto outputs_res = module_->execute(method_name, inputs);
56+
auto outputs_res = module_->execute("forward", inputs);
6257
ET_CHECK_OK_OR_RETURN_ERROR(outputs_res.error());
6358

6459
auto update_err = io_manager_->update_decode(outputs_res.get());

0 commit comments

Comments
 (0)