2020#include " constants.h"
2121#include " util.h"
2222#include < algorithm>
23- #include < cmath>
24- #include < cstdio>
2523#include < cstring>
2624#include < rnexecutorch/Log.h>
2725#include < string>
@@ -44,8 +42,6 @@ MultimodalPrefiller::MultimodalPrefiller(
4442Result<uint64_t >
4543MultimodalPrefiller::prefill (const std::vector<MultimodalInput> &inputs,
4644 int64_t &start_pos) {
47- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa multimodal prefill" );
48-
4945 const bool has_ple = decoder_runner_->has_ple ();
5046 const long t_prefill_begin = time_in_ms ();
5147
@@ -138,11 +134,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
138134 int audio_calls = 0 ;
139135
140136 for (const auto &input : inputs) {
141-
142- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa input type" ,
143- input.is_audio () ? " audio" : " no audio" ,
144- input.is_image () ? " image" : " no image" );
145-
146137 if (input.is_image ()) {
147138 ET_CHECK_OR_RETURN_ERROR (image_encoder_ != nullptr , InvalidState,
148139 " No image encoder registered" );
@@ -156,9 +147,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
156147 ET_CHECK_OR_RETURN_ERROR (audio_encoder_ != nullptr , InvalidState,
157148 " No audio encoder registered" );
158149 const long t_aud_begin = time_in_ms ();
159- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa encoding audio" );
160150 auto enc = audio_encoder_->encode (input);
161- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa encoded audio" );
162151 ET_CHECK_OK_OR_RETURN_ERROR (enc.error (), " Audio encoding failed" );
163152 audio_encode_ms += time_in_ms () - t_aud_begin;
164153 audio_calls += 1 ;
@@ -173,8 +162,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
173162 audio_tensor.dim ());
174163 const int64_t num_audio = static_cast <int64_t >(audio_tensor.size (1 ));
175164 const int64_t audio_hidden = static_cast <int64_t >(audio_tensor.size (2 ));
176- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa encoded audio" ,
177- num_audio, audio_hidden);
178165 ET_CHECK_OR_RETURN_ERROR (num_audio > 0 , InvalidState,
179166 " Audio encoder produced 0 tokens" );
180167 std::vector<uint8_t > bytes (audio_tensor.nbytes ());
@@ -184,62 +171,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
184171 AudioSlot{std::move (bytes), audio_tensor.scalar_type (),
185172 static_cast <int64_t >(ids.size ()), num_audio, audio_hidden});
186173 ids.insert (ids.end (), static_cast <size_t >(num_audio), 0 );
187-
188- // Diagnostic: dump audio encoder output magnitude + first values so
189- // the user can see in the JS console whether the encoder is producing
190- // real embeddings or zeros/noise. slot.bytes can be either fp32 or
191- // fp16 depending on exporter; handle both.
192- {
193- const AudioSlot &slot = audio_slots.back ();
194- const size_t nrows = static_cast <size_t >(slot.num_audio );
195- const size_t hidden = static_cast <size_t >(slot.audio_hidden );
196- const size_t nfloats = nrows * hidden;
197- const size_t bytes_per_elem =
198- nfloats > 0 ? slot.bytes .size () / nfloats : 0 ;
199- float maxabs = 0 .0f ;
200- double sumsq = 0.0 ;
201- std::string head;
202- const size_t headN = std::min<size_t >(16 , nfloats);
203- if (slot.dtype == ::executorch::aten::ScalarType::Float) {
204- const float *e = reinterpret_cast <const float *>(slot.bytes .data ());
205- for (size_t i = 0 ; i < nfloats; ++i) {
206- const float a = std::fabs (e[i]);
207- if (a > maxabs)
208- maxabs = a;
209- sumsq += static_cast <double >(e[i]) * static_cast <double >(e[i]);
210- }
211- for (size_t i = 0 ; i < headN; ++i) {
212- if (i)
213- head += " , " ;
214- head += std::to_string (e[i]);
215- }
216- } else if (slot.dtype == ::executorch::aten::ScalarType::Half) {
217- const auto *e = reinterpret_cast <const ::executorch::aten::Half *>(
218- slot.bytes .data ());
219- for (size_t i = 0 ; i < nfloats; ++i) {
220- const float v = static_cast <float >(e[i]);
221- const float a = std::fabs (v);
222- if (a > maxabs)
223- maxabs = a;
224- sumsq += static_cast <double >(v) * static_cast <double >(v);
225- }
226- for (size_t i = 0 ; i < headN; ++i) {
227- if (i)
228- head += " , " ;
229- head += std::to_string (static_cast <float >(e[i]));
230- }
231- } else {
232- head = " <unsupported dtype>" ;
233- }
234- const double rms =
235- nfloats > 0 ? std::sqrt (sumsq / static_cast <double >(nfloats)) : 0.0 ;
236- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info,
237- " kappa [AudioEmbed] num_audio=" , nrows,
238- " hidden=" , hidden, " slot_start=" , slot.slot_start ,
239- " dtype=" , static_cast <int >(slot.dtype ),
240- " bytes_per_elem=" , bytes_per_elem,
241- " maxabs=" , maxabs, " rms=" , rms, " first16=" , head);
242- }
243174 } else if (input.is_text () || input.is_tokens ()) {
244175 std::vector<uint64_t > tokens;
245176 if (input.is_text ()) {
@@ -279,42 +210,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
279210 ids.resize (static_cast <size_t >(max_seq_len), 0 );
280211 }
281212
282- // ------------------------------------------------------------
283- // Splice diagnostics: dump the fused id stream + slot map so the JS
284- // console can confirm placeholder positions (audio/image use id=0)
285- // line up with where the encoder outputs will be spliced in pass 2/2b.
286- // ------------------------------------------------------------
287- {
288- std::string ids_dump;
289- const size_t dump_n = std::min<size_t >(ids.size (), size_t {512 });
290- ids_dump.reserve (dump_n * 6 );
291- for (size_t i = 0 ; i < dump_n; ++i) {
292- if (i)
293- ids_dump += " ," ;
294- ids_dump += std::to_string (ids[i]);
295- }
296- if (ids.size () > dump_n)
297- ids_dump += " ,…" ;
298- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info,
299- " [Splice] total_len=" , total_len, " ids_buf=" , ids.size (),
300- " image_slots=" , image_slots.size (),
301- " audio_slots=" , audio_slots.size (), " ids[0.." , dump_n,
302- " ]=" , ids_dump);
303- for (size_t i = 0 ; i < image_slots.size (); ++i) {
304- const auto &s = image_slots[i];
305- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " [Splice] image_slot[" ,
306- i, " ] start=" , s.slot_start , " num=" , s.num_visual );
307- }
308- for (size_t i = 0 ; i < audio_slots.size (); ++i) {
309- const auto &s = audio_slots[i];
310- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " [Splice] audio_slot[" ,
311- i, " ] start=" , s.slot_start , " num=" , s.num_audio ,
312- " hidden=" , s.audio_hidden ,
313- " dtype=" , static_cast <int >(s.dtype ),
314- " bytes=" , s.bytes .size ());
315- }
316- }
317-
318213 // ------------------------------------------------------------
319214 // Single token_embedding call over the fused id buffer.
320215 // ------------------------------------------------------------
@@ -393,7 +288,10 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
393288 const long t_vis_begin = time_in_ms ();
394289 auto encode_result = image_encoder_->encode (*slot.input );
395290 ET_CHECK_OK_OR_RETURN_ERROR (encode_result.error (), " Image encoding failed" );
396- encoder_output = *encode_result;
291+ vision_total_ms += time_in_ms () - t_vis_begin;
292+ vision_calls += 1 ;
293+ auto encoder_output = *encode_result;
294+ auto vision_tensor = encoder_output.toTensor ();
397295
398296 const auto vision_dtype = vision_tensor.scalar_type ();
399297 const size_t visual_elems =
@@ -433,44 +331,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
433331 // byte snapshot taken at encode time so post-encode execute() calls can't
434332 // invalidate slot state. Same dtype-conversion matrix as vision.
435333 // ------------------------------------------------------------
436-
437- // Diagnostic helper: stringify the first N elements of a byte buffer
438- // interpreted as the given scalar dtype (Float or Half supported). Used
439- // for the before/src/after snapshots logged per audio slot so the user
440- // can verify (a) the destination held pad-token embeds prior to splice,
441- // (b) the source bytes carry real audio encoder output, and (c) the
442- // destination contains the audio values post-splice.
443- auto stringify_floats = [](const uint8_t *p, size_t n,
444- ::executorch::aten::ScalarType dt) -> std::string {
445- std::string out;
446- out.reserve (n * 10 );
447- for (size_t i = 0 ; i < n; ++i) {
448- if (i)
449- out += " ," ;
450- float v = 0 .0f ;
451- if (dt == ::executorch::aten::ScalarType::Float) {
452- v = reinterpret_cast <const float *>(p)[i];
453- } else if (dt == ::executorch::aten::ScalarType::Half) {
454- v = static_cast <float >(
455- reinterpret_cast <const ::executorch::aten::Half *>(p)[i]);
456- }
457- char buf[24 ];
458- std::snprintf (buf, sizeof (buf), " %.4f" , v);
459- out += buf;
460- }
461- return out;
462- };
463-
464- rnexecutorch::log (
465- rnexecutorch::LOG_LEVEL ::Info,
466- " kappa [Splice] pass 2b begin: audio_slots=" , audio_slots.size (),
467- " embeds_buf_bytes=" , embeds_buf.size (), " hidden=" , hidden,
468- " embeds_dtype=" , static_cast <int >(embeds_dtype),
469- " embeds_elem_size=" , embeds_elem_size);
470-
471334 for (auto &slot : audio_slots) {
472- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa splice loop!!!" );
473-
474335 ET_CHECK_OR_RETURN_ERROR (
475336 slot.audio_hidden == static_cast <int64_t >(hidden), InvalidState,
476337 " audio encoder hidden %lld != text_embed hidden %lld" ,
@@ -494,16 +355,6 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
494355 static_cast <size_t >(hidden) *
495356 embeds_elem_size;
496357
497- // Snapshot the first row before splice. These bytes should hold the
498- // pad-token (id=0) embedding because ids.insert pushed zeros into
499- // these slot positions before the token_embedding call. If `before`
500- // already equals `src` you are looking at a re-run; if `after` differs
501- // from `src` the splice/dtype-cast path is wrong.
502- const size_t dbg_n = std::min<size_t >(8 , static_cast <size_t >(hidden));
503- const std::string before_str = stringify_floats (dst, dbg_n, embeds_dtype);
504- const std::string src_str =
505- stringify_floats (slot.bytes .data (), dbg_n, audio_dtype);
506-
507358 if (audio_dtype == embeds_dtype) {
508359 std::memcpy (dst, slot.bytes .data (), audio_elems * embeds_elem_size);
509360 } else if (audio_dtype == ::executorch::aten::ScalarType::Float &&
@@ -527,40 +378,7 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
527378 " unsupported audio/text dtype pair: audio=%hhd text=%hhd" ,
528379 static_cast <int8_t >(audio_dtype), static_cast <int8_t >(embeds_dtype));
529380 }
530-
531- // Post-splice: re-read the same dst row + compute magnitude over the
532- // entire spliced region so the user can confirm the bytes flipped to
533- // real audio embeds (post should equal src, post_maxabs > 0 always).
534- const std::string after_str = stringify_floats (dst, dbg_n, embeds_dtype);
535- float maxabs = 0 .0f ;
536- double sumsq = 0.0 ;
537- for (size_t i = 0 ; i < audio_elems; ++i) {
538- float v = 0 .0f ;
539- if (embeds_dtype == ::executorch::aten::ScalarType::Float) {
540- v = reinterpret_cast <const float *>(dst)[i];
541- } else if (embeds_dtype == ::executorch::aten::ScalarType::Half) {
542- v = static_cast <float >(
543- reinterpret_cast <const ::executorch::aten::Half *>(dst)[i]);
544- }
545- const float av = std::fabs (v);
546- if (av > maxabs)
547- maxabs = av;
548- sumsq += static_cast <double >(v) * static_cast <double >(v);
549- }
550- const double rms = audio_elems > 0
551- ? std::sqrt (sumsq / static_cast <double >(audio_elems))
552- : 0.0 ;
553-
554- rnexecutorch::log (
555- rnexecutorch::LOG_LEVEL ::Info,
556- " kappa [Splice] audio slot start=" , slot.slot_start ,
557- " num=" , slot.num_audio , " dtype src=" , static_cast <int >(audio_dtype),
558- " dst=" , static_cast <int >(embeds_dtype), " pre[0.." , dbg_n,
559- " ]=" , before_str, " src[0.." , dbg_n, " ]=" , src_str, " post[0.." , dbg_n,
560- " ]=" , after_str, " post_maxabs=" , maxabs, " post_rms=" , rms);
561381 }
562- rnexecutorch::log (rnexecutorch::LOG_LEVEL ::Info, " kappa post splice" ,
563- embeds_buf);
564382
565383 // ------------------------------------------------------------
566384 // Chunked text_decoder calls.
0 commit comments