minor refactor

Author: mkopcins

packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h

@@ -1,9 +1,11 @@
 #pragma once
 
+#include "rnexecutorch/Log.h"
 #include <codecvt>
 #include <cstdint>
 #include <set>
 #include <span>
+#include <string>
 #include <type_traits>
 #include <unordered_map>
 #include <variant>
@@ -319,28 +321,28 @@ getValue<std::span<uint64_t>>(const jsi::Value &val, jsi::Runtime &runtime) {
   return getTypedArrayAsSpan<uint64_t>(val, runtime);
 }
 
-template <typename T>
-inline void assignIfPresent(T &field, const jsi::Object &obj,
-                            jsi::Runtime &runtime, const char *name) {
-  if (!obj.hasProperty(runtime, name))
-    return;
-  jsi::Value v = obj.getProperty(runtime, name);
-  if (v.isUndefined() || v.isNull())
-    return;
-  field = getValue<T>(v, runtime);
-}
-
 template <>
 inline models::llm::MultimodalInputs
 getValue<models::llm::MultimodalInputs>(const jsi::Value &val,
                                         jsi::Runtime &runtime) {
   models::llm::MultimodalInputs multimodalInputs;
   jsi::Object obj = val.asObject(runtime);
-  assignIfPresent(multimodalInputs.images->paths, obj, runtime, "imagePaths");
-  assignIfPresent(multimodalInputs.images->token, obj, runtime, "imageToken");
-  assignIfPresent(multimodalInputs.audios->waveforms, obj, runtime,
-                  "audioWaveforms");
-  assignIfPresent(multimodalInputs.audios->token, obj, runtime, "audioToken");
+
+  jsi::Value v = obj.getProperty(runtime, "imageToken");
+  if (!v.isUndefined() && !v.isNull()) {
+    auto &images = multimodalInputs.images.emplace();
+    images.token = getValue<std::string>(v, runtime);
+    v = obj.getProperty(runtime, "imagePaths");
+    images.paths = getValue<std::vector<std::string>>(v, runtime);
+  }
+  v = obj.getProperty(runtime, "audioToken");
+  if (!v.isUndefined() && !v.isNull()) {
+    auto &audios = multimodalInputs.audios.emplace();
+    audios.token = getValue<std::string>(v, runtime);
+    v = obj.getProperty(runtime, "audioWaveforms");
+    audios.waveforms = getValue<std::vector<std::vector<float>>>(v, runtime);
+  }
+
   return multimodalInputs;
 }
 

packages/react-native-executorch/common/rnexecutorch/host_objects/ModelHostObject.h

@@ -281,9 +281,11 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
         return jsi_conversion::getJsiValue(std::move(result), runtime);
       }
     } catch (const RnExecutorchError &e) {
+      rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
       throw jsi::JSError(runtime, makeRnExecutorchErrorValue(
                                       runtime, e.getNumericCode(), e.what()));
     } catch (const std::exception &e) {
+      rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
       throw jsi::JSError(runtime, e.what());
     } catch (...) {
       throw jsi::JSError(runtime, "Unknown error in synchronous function");
@@ -348,9 +350,11 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
         return jsi_conversion::getJsiValue(std::move(result), runtime);
       }
     } catch (const RnExecutorchError &e) {
+      rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
       throw jsi::JSError(runtime, makeRnExecutorchErrorValue(
                                       runtime, e.getNumericCode(), e.what()));
     } catch (const std::exception &e) {
+      rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
       throw jsi::JSError(runtime, e.what());
     } catch (...) {
       throw jsi::JSError(runtime, "Unknown error in vision function");
@@ -411,6 +415,8 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
                           });
                     }
                   } catch (const RnExecutorchError &e) {
+                    rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa",
+                                      e.what());
                     auto code = e.getNumericCode();
                     auto msg = std::string(e.what());
                     callInvoker->invokeAsync(
@@ -420,6 +426,8 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
                         });
                     return;
                   } catch (const std::exception &e) {
+                    rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa",
+                                      e.what());
                     callInvoker->invokeAsync([e = std::move(e), promise]() {
                       promise->reject(std::string(e.what()));
                     });

packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp

@@ -1,4 +1,5 @@
 #include "LLM.h"
+#include "rnexecutorch/models/llm/Types.h"
 
 #include <executorch/extension/tensor/tensor.h>
 #include <filesystem>
@@ -87,8 +88,8 @@ std::string LLM::generateMultimodal(std::string prompt,
     throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
                             "This model does not support multimodal input.");
   }
-  if (mutlimodalInputs.imageToken.empty() &&
-      mutlimodalInputs.audioToken.empty()) {
+  if (!mutlimodalInputs.images.has_value() &&
+      !mutlimodalInputs.audios.has_value()) {
     throw RnExecutorchError(
         RnExecutorchErrorCode::InvalidUserInput,
         "At least one of imageToken/audioToken must be non-empty");
@@ -99,12 +100,12 @@ std::string LLM::generateMultimodal(std::string prompt,
   std::vector<llm::MultimodalInput> inputs;
   size_t imageIdx = 0, audioIdx = 0, pos = 0;
   while (pos < prompt.size()) {
-    size_t imgAt = mutlimodalInputs.images
-                       ? std::string::npos
-                       : prompt.find(mutlimodalInputs.images->token, pos);
-    size_t audAt = mutlimodalInputs.audios
-                       ? std::string::npos
-                       : prompt.find(mutlimodalInputs.audios->token, pos);
+    size_t imgAt = mutlimodalInputs.images.has_value()
+                       ? prompt.find(mutlimodalInputs.images.value().token, pos)
+                       : std::string::npos;
+    size_t audAt = mutlimodalInputs.audios.has_value()
+                       ? prompt.find(mutlimodalInputs.audios.value().token, pos)
+                       : std::string::npos;
     if (imgAt == std::string::npos && audAt == std::string::npos) {
       inputs.push_back(llm::make_text_input(prompt.substr(pos)));
       break;
@@ -116,27 +117,30 @@ std::string LLM::generateMultimodal(std::string prompt,
       inputs.push_back(llm::make_text_input(prompt.substr(pos, at - pos)));
     }
     if (imageFirst) {
-      if (imageIdx >= mutlimodalInputs.images->paths.size()) {
+      auto &images = mutlimodalInputs.images.value();
+      if (imageIdx >= images.paths.size()) {
         throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
-                                "More '" + mutlimodalInputs.images->token +
+                                "More '" + images.token +
                                     "' placeholders than image paths");
       }
-      inputs.push_back(
-          llm::make_image_input(mutlimodalInputs.images->paths[imageIdx++]));
-      pos = at + mutlimodalInputs.images->token.size();
+      inputs.push_back(llm::make_image_input(images.paths[imageIdx++]));
+      pos = at + images.token.size();
     } else {
-      if (audioIdx >= mutlimodalInputs.audios->waveforms.size()) {
+      auto &audios = mutlimodalInputs.audios.value();
+      if (audioIdx >= audios.waveforms.size()) {
         throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
-                                "More '" + mutlimodalInputs.audioToken +
+                                "More '" + audios.token +
                                     "' placeholders than audio waveforms");
       }
-      inputs.push_back(llm::make_audio_input(
-          std::move(mutlimodalInputs.audios->waveforms[audioIdx++])));
-      pos = at + mutlimodalInputs.audios->token.size();
+      inputs.push_back(
+          llm::make_audio_input(std::move(audios.waveforms[audioIdx++])));
+      pos = at + audios.token.size();
     }
   }
-  if (imageIdx < mutlimodalInputs.images->paths.size() ||
-      audioIdx < mutlimodalInputs.audios->waveforms.size()) {
+  if ((mutlimodalInputs.images.has_value() &&
+       imageIdx < mutlimodalInputs.images.value().paths.size()) ||
+      (mutlimodalInputs.audios.has_value() &&
+       audioIdx < mutlimodalInputs.audios.value().waveforms.size())) {
     throw RnExecutorchError(
         RnExecutorchErrorCode::InvalidUserInput,
         "More image/audio paths provided than placeholders in prompt");

packages/react-native-executorch/common/runner/multimodal_prefiller.cpp

@@ -11,7 +11,6 @@
 
 #include "multimodal_prefiller.h"
 #include "constants.h"
-#include "runner/multimodal_decoder_runner.h"
 #include "util.h"
 #include <algorithm>
 #include <cstring>
@@ -164,6 +163,7 @@ auto MultimodalPrefiller::encodeAudio(
         "unsupported audio/text dtype pair: audio=%hhd text=%hhd",
         static_cast<int8_t>(audio_dtype), static_cast<int8_t>(embeds_dtype));
   }
+  return ::executorch::runtime::Error::Ok;
 }
 
 auto MultimodalPrefiller::encodeImages(
@@ -205,6 +205,7 @@ auto MultimodalPrefiller::encodeImages(
         "unsupported vision/text dtype pair: vision=%hhd text=%hhd",
         static_cast<int8_t>(vision_dtype), static_cast<int8_t>(embeds_dtype));
   }
+  return ::executorch::runtime::Error::Ok;
 }
 
 auto MultimodalPrefiller::initializePLE(auto &embed_outputs, auto total_len,
@@ -225,6 +226,7 @@ auto MultimodalPrefiller::initializePLE(auto &embed_outputs, auto total_len,
   ple_embeddings.ple_tok_buf.resize(prefix_bytes);
   std::memcpy(ple_embeddings.ple_tok_buf.data(),
               full_ple_tok.mutable_data_ptr(), prefix_bytes);
+  return ::executorch::runtime::Error::Ok;
 }
 
 auto MultimodalPrefiller::prefillChunk(std::vector<EValue> &last_outs,
@@ -268,6 +270,7 @@ auto MultimodalPrefiller::prefillChunk(std::vector<EValue> &last_outs,
                                                        EValue(*pos_chunk)});
   ET_CHECK_OK_OR_RETURN_ERROR(res.error());
   last_outs = std::move(*res);
+  return ::executorch::runtime::Error::Ok;
 }
 
 Result<uint64_t>

packages/react-native-executorch/common/runner/multimodal_prefiller.h

@@ -19,28 +19,12 @@
 namespace executorch::extension::llm {
 
 namespace {
-// ------------------------------------------------------------
-// Pass 1: build a fused input_ids buffer spanning all inputs.
-//
-// Image positions use pad_token_id (placeholder_id is rewritten to 0 before
-// PLE lookup). The decoder embeds at those positions are then overwritten
-// with the vision encoder output in pass 2.
-// ------------------------------------------------------------
 struct ImageSlot {
   const MultimodalInput *input; // non-owning, valid for duration of call
   int64_t slot_start;
   int64_t num_visual;
 };
-// Audio tokens are dynamic per clip, so we encode first and remember a
-// BYTE SNAPSHOT of the encoder output + count + dtype; pass 2 splices
-// from the snapshot.
-//
-// We can NOT stash the EValue here. EValue holds an aten::Tensor which is
-// just a TensorImpl*; `Method::get_output(i)` returns `const EValue&` to
-// Method-internal storage and Module::execute copies that EValue into the
-// returned vector. The copy shares the underlying TensorImpl, so a later
-// execute() on the same method — a second audio input in this prefill would
-// overwrite.
+
 struct AudioSlot {
   std::vector<uint8_t> bytes;
   ::executorch::aten::ScalarType dtype;

packages/react-native-executorch/common/runner/sampler.h

@@ -29,6 +29,7 @@ namespace executorch {
 namespace extension {
 namespace llm {
 // A simple llama2 sampler.
+struct GenerationConfig;
 
 inline constexpr auto kTopp = 0.9f;
 

packages/react-native-executorch/common/runner/text_decoder_runner.cpp

@@ -101,9 +101,7 @@ int32_t TextDecoderRunner::logits_to_token(
           auto num_tokens = logits_tensor.size(1);
           logits += (num_tokens - 1) * vocab_size;
         }
-        Sampler sampler(vocab_size, config_.temperature, config_.topp,
-                        static_cast<unsigned long long>(std::time(nullptr)),
-                        config_.min_p, config_.repetition_penalty);
+        Sampler sampler(vocab_size, config_);
         result = sampler.sample(logits, recent_tokens);
       });
   return result;

packages/react-native-executorch/common/runner/util.h

@@ -8,7 +8,6 @@
 
 #pragma once
 #include "constants.h"
-#include "text_prefiller.h"
 #include <cctype>
 #include <executorch/extension/module/module.h>
 #include <executorch/extension/tensor/tensor.h>

packages/react-native-executorch/src/controllers/LLMController.ts

@@ -305,9 +305,9 @@ export class LLMController {
           input,
           this.onToken,
           {
-            imagePaths: hasImages ? imagePaths!.map(normalizeImagePath) : [],
+            imagePaths: hasImages ? imagePaths!.map(normalizeImagePath) : null,
             imageToken: hasImages ? this.getImageToken() : null,
-            audioWaveforms: hasAudio ? audioWaveforms! : [],
+            audioWaveforms: hasAudio ? audioWaveforms! : null,
             audioToken: hasAudio ? this.getAudioToken() : null,
           }
         );