Skip to content

Commit b523e57

Browse files
mkopcinsmkopcins
committed
minor prefiller refactor
1 parent b938284 commit b523e57

13 files changed

Lines changed: 380 additions & 364 deletions

File tree

‎apps/llm/app/multimodal_llm/index.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ function MultimodalLLMScreen() {
6464
const [error, setError] = useState<string | null>(null);
6565

6666
const vlm = useLLM({
67-
model: models.llm.gemma4_e2b(),
67+
model: models.llm.gemma4_e2b_multimodal(),
6868
});
6969
const tokenCount = vlm.isReady ? vlm.getGeneratedTokenCount() : 0;
7070
const { stats, onMessageSend } = useLLMStats(

‎packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#pragma once
22

3+
#include "rnexecutorch/Log.h"
34
#include <codecvt>
45
#include <cstdint>
56
#include <set>
67
#include <span>
8+
#include <string>
79
#include <type_traits>
810
#include <unordered_map>
911
#include <variant>
@@ -319,28 +321,28 @@ getValue<std::span<uint64_t>>(const jsi::Value &val, jsi::Runtime &runtime) {
319321
return getTypedArrayAsSpan<uint64_t>(val, runtime);
320322
}
321323

322-
template <typename T>
323-
inline void assignIfPresent(T &field, const jsi::Object &obj,
324-
jsi::Runtime &runtime, const char *name) {
325-
if (!obj.hasProperty(runtime, name))
326-
return;
327-
jsi::Value v = obj.getProperty(runtime, name);
328-
if (v.isUndefined() || v.isNull())
329-
return;
330-
field = getValue<T>(v, runtime);
331-
}
332-
333324
template <>
334325
inline models::llm::MultimodalInputs
335326
getValue<models::llm::MultimodalInputs>(const jsi::Value &val,
336327
jsi::Runtime &runtime) {
337328
models::llm::MultimodalInputs multimodalInputs;
338329
jsi::Object obj = val.asObject(runtime);
339-
assignIfPresent(multimodalInputs.imagePaths, obj, runtime, "imagePaths");
340-
assignIfPresent(multimodalInputs.imageToken, obj, runtime, "imageToken");
341-
assignIfPresent(multimodalInputs.audioWaveforms, obj, runtime,
342-
"audioWaveforms");
343-
assignIfPresent(multimodalInputs.audioToken, obj, runtime, "audioToken");
330+
331+
jsi::Value v = obj.getProperty(runtime, "imageToken");
332+
if (!v.isUndefined() && !v.isNull()) {
333+
auto &images = multimodalInputs.images.emplace();
334+
images.token = getValue<std::string>(v, runtime);
335+
v = obj.getProperty(runtime, "imagePaths");
336+
images.paths = getValue<std::vector<std::string>>(v, runtime);
337+
}
338+
v = obj.getProperty(runtime, "audioToken");
339+
if (!v.isUndefined() && !v.isNull()) {
340+
auto &audios = multimodalInputs.audios.emplace();
341+
audios.token = getValue<std::string>(v, runtime);
342+
v = obj.getProperty(runtime, "audioWaveforms");
343+
audios.waveforms = getValue<std::vector<std::vector<float>>>(v, runtime);
344+
}
345+
344346
return multimodalInputs;
345347
}
346348

‎packages/react-native-executorch/common/rnexecutorch/host_objects/ModelHostObject.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,9 +284,11 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
284284
return jsi_conversion::getJsiValue(std::move(result), runtime);
285285
}
286286
} catch (const RnExecutorchError &e) {
287+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
287288
throw jsi::JSError(runtime, makeRnExecutorchErrorValue(
288289
runtime, e.getNumericCode(), e.what()));
289290
} catch (const std::exception &e) {
291+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
290292
throw jsi::JSError(runtime, e.what());
291293
} catch (...) {
292294
throw jsi::JSError(runtime, "Unknown error in synchronous function");
@@ -351,9 +353,11 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
351353
return jsi_conversion::getJsiValue(std::move(result), runtime);
352354
}
353355
} catch (const RnExecutorchError &e) {
356+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
354357
throw jsi::JSError(runtime, makeRnExecutorchErrorValue(
355358
runtime, e.getNumericCode(), e.what()));
356359
} catch (const std::exception &e) {
360+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa", e.what());
357361
throw jsi::JSError(runtime, e.what());
358362
} catch (...) {
359363
throw jsi::JSError(runtime, "Unknown error in vision function");
@@ -414,6 +418,8 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
414418
});
415419
}
416420
} catch (const RnExecutorchError &e) {
421+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa",
422+
e.what());
417423
auto code = e.getNumericCode();
418424
auto msg = std::string(e.what());
419425
callInvoker->invokeAsync(
@@ -423,6 +429,8 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
423429
});
424430
return;
425431
} catch (const std::exception &e) {
432+
rnexecutorch::log(rnexecutorch::LOG_LEVEL::Info, "kappa",
433+
e.what());
426434
callInvoker->invokeAsync([e = std::move(e), promise]() {
427435
promise->reject(std::string(e.what()));
428436
});

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp‎

Lines changed: 24 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "LLM.h"
2+
#include "rnexecutorch/models/llm/Types.h"
23

34
#include <executorch/extension/tensor/tensor.h>
45
#include <filesystem>
@@ -87,8 +88,8 @@ std::string LLM::generateMultimodal(std::string prompt,
8788
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
8889
"This model does not support multimodal input.");
8990
}
90-
if (mutlimodalInputs.imageToken.empty() &&
91-
mutlimodalInputs.audioToken.empty()) {
91+
if (!mutlimodalInputs.images.has_value() &&
92+
!mutlimodalInputs.audios.has_value()) {
9293
throw RnExecutorchError(
9394
RnExecutorchErrorCode::InvalidUserInput,
9495
"At least one of imageToken/audioToken must be non-empty");
@@ -99,12 +100,12 @@ std::string LLM::generateMultimodal(std::string prompt,
99100
std::vector<llm::MultimodalInput> inputs;
100101
size_t imageIdx = 0, audioIdx = 0, pos = 0;
101102
while (pos < prompt.size()) {
102-
size_t imgAt = mutlimodalInputs.imageToken.empty()
103-
? std::string::npos
104-
: prompt.find(mutlimodalInputs.imageToken, pos);
105-
size_t audAt = mutlimodalInputs.audioToken.empty()
106-
? std::string::npos
107-
: prompt.find(mutlimodalInputs.audioToken, pos);
103+
size_t imgAt = mutlimodalInputs.images.has_value()
104+
? prompt.find(mutlimodalInputs.images.value().token, pos)
105+
: std::string::npos;
106+
size_t audAt = mutlimodalInputs.audios.has_value()
107+
? prompt.find(mutlimodalInputs.audios.value().token, pos)
108+
: std::string::npos;
108109
if (imgAt == std::string::npos && audAt == std::string::npos) {
109110
inputs.push_back(llm::make_text_input(prompt.substr(pos)));
110111
break;
@@ -116,27 +117,30 @@ std::string LLM::generateMultimodal(std::string prompt,
116117
inputs.push_back(llm::make_text_input(prompt.substr(pos, at - pos)));
117118
}
118119
if (imageFirst) {
119-
if (imageIdx >= mutlimodalInputs.imagePaths.size()) {
120+
auto &images = mutlimodalInputs.images.value();
121+
if (imageIdx >= images.paths.size()) {
120122
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
121-
"More '" + mutlimodalInputs.imageToken +
123+
"More '" + images.token +
122124
"' placeholders than image paths");
123125
}
124-
inputs.push_back(
125-
llm::make_image_input(mutlimodalInputs.imagePaths[imageIdx++]));
126-
pos = at + mutlimodalInputs.imageToken.size();
126+
inputs.push_back(llm::make_image_input(images.paths[imageIdx++]));
127+
pos = at + images.token.size();
127128
} else {
128-
if (audioIdx >= mutlimodalInputs.audioWaveforms.size()) {
129+
auto &audios = mutlimodalInputs.audios.value();
130+
if (audioIdx >= audios.waveforms.size()) {
129131
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
130-
"More '" + mutlimodalInputs.audioToken +
132+
"More '" + audios.token +
131133
"' placeholders than audio waveforms");
132134
}
133-
inputs.push_back(llm::make_audio_input(
134-
std::move(mutlimodalInputs.audioWaveforms[audioIdx++])));
135-
pos = at + mutlimodalInputs.audioToken.size();
135+
inputs.push_back(
136+
llm::make_audio_input(std::move(audios.waveforms[audioIdx++])));
137+
pos = at + audios.token.size();
136138
}
137139
}
138-
if (imageIdx < mutlimodalInputs.imagePaths.size() ||
139-
audioIdx < mutlimodalInputs.audioWaveforms.size()) {
140+
if ((mutlimodalInputs.images.has_value() &&
141+
imageIdx < mutlimodalInputs.images.value().paths.size()) ||
142+
(mutlimodalInputs.audios.has_value() &&
143+
audioIdx < mutlimodalInputs.audios.value().waveforms.size())) {
140144
throw RnExecutorchError(
141145
RnExecutorchErrorCode::InvalidUserInput,
142146
"More image/audio paths provided than placeholders in prompt");

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,7 @@ class LLM : public BaseModel {
2323

2424
std::string generate(std::string prompt,
2525
std::shared_ptr<jsi::Function> callback);
26-
// Audio variant: `audioWaveforms` is a parallel vector of fp32 mono 16 kHz
27-
// PCM buffers (decoded upstream, same contract as SpeechToText::transcribe).
28-
// The prompt is scanned for `imageToken` and/or `audioToken` placeholders;
29-
// each placeholder consumes the next entry from its respective vector in
30-
// order. Either set of paths/waveforms/token may be empty.
26+
3127
std::string generateMultimodal(std::string prompt,
3228
std::shared_ptr<jsi::Function> callback,
3329
MultimodalInputs mutlimodalInputs = {});

‎packages/react-native-executorch/common/rnexecutorch/models/llm/Types.h‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,19 @@
44
#include <vector>
55

66
namespace rnexecutorch::models::llm {
7+
struct ImageInputs {
8+
std::vector<std::string> paths;
9+
std::string token;
10+
};
11+
12+
struct AudioInputs {
13+
std::vector<std::vector<float>> waveforms;
14+
std::string token;
15+
};
16+
717
struct MultimodalInputs {
8-
std::vector<std::string> imagePaths;
9-
std::string imageToken;
10-
std::vector<std::vector<float>> audioWaveforms;
11-
std::string audioToken;
18+
std::optional<ImageInputs> images;
19+
std::optional<AudioInputs> audios;
1220
};
13-
using Token = uint64_t;
1421

15-
} // namespace rnexecutorch::models::llm
22+
} // namespace rnexecutorch::models::llm

‎packages/react-native-executorch/common/runner/encoders/audio_encoder.cpp‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@ using ::executorch::runtime::EValue;
2020
using ::executorch::runtime::Result;
2121

2222
namespace {
23+
constexpr int32_t kSamplingRate = 16e3;
24+
constexpr int32_t kMaxLengthSeconds = 30;
2325
constexpr int32_t kSamplesPerBlock = 7680;
2426
constexpr int64_t kAudioBlockKMin = 1;
25-
constexpr int64_t kAudioBlockKMax = 62;
27+
constexpr int64_t kAudioBlockKMax =
28+
kSamplingRate * kMaxLengthSeconds / kSamplesPerBlock;
2629
} // namespace
2730

2831
AudioEncoder::AudioEncoder(::executorch::extension::Module &module)
@@ -70,15 +73,8 @@ Result<EValue> AudioEncoder::encode(const MultimodalInput &input) {
7073
ET_CHECK_OR_RETURN_ERROR(
7174
k_blocks >= kAudioBlockKMin && k_blocks <= kAudioBlockKMax,
7275
InvalidArgument,
73-
"AudioEncoder: waveform of %lld samples needs k_blocks=%lld; "
74-
"audio_encoder accepts k in [%lld, %lld] (block=%d samples; max %.2f s "
75-
"@ 16 kHz)",
76-
static_cast<long long>(n_valid), static_cast<long long>(k_blocks),
77-
static_cast<long long>(kAudioBlockKMin),
78-
static_cast<long long>(kAudioBlockKMax),
79-
static_cast<int>(kSamplesPerBlock),
80-
static_cast<double>(kSamplesPerBlock) *
81-
static_cast<double>(kAudioBlockKMax) / 16000.0);
76+
"AudioEncoder: waveform of %lld samples needs k_blocks=%lld.",
77+
static_cast<long long>(n_valid), static_cast<long long>(k_blocks));
8278
const int64_t n_padded = k_blocks * kSamplesPerBlock;
8379

8480
// Own the padded waveform for the lifetime of this call; from_blob below

0 commit comments

Comments
 (0)