Skip to content

Commit fe51028

Browse files
mkopcinsmkopcins
committed
minor prefiller refactor
1 parent 3be9c42 commit fe51028

7 files changed

Lines changed: 355 additions & 338 deletions

File tree

‎packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -336,11 +336,11 @@ getValue<models::llm::MultimodalInputs>(const jsi::Value &val,
336336
jsi::Runtime &runtime) {
337337
models::llm::MultimodalInputs multimodalInputs;
338338
jsi::Object obj = val.asObject(runtime);
339-
assignIfPresent(multimodalInputs.imagePaths, obj, runtime, "imagePaths");
340-
assignIfPresent(multimodalInputs.imageToken, obj, runtime, "imageToken");
341-
assignIfPresent(multimodalInputs.audioWaveforms, obj, runtime,
339+
assignIfPresent(multimodalInputs.images->paths, obj, runtime, "imagePaths");
340+
assignIfPresent(multimodalInputs.images->token, obj, runtime, "imageToken");
341+
assignIfPresent(multimodalInputs.audios->waveforms, obj, runtime,
342342
"audioWaveforms");
343-
assignIfPresent(multimodalInputs.audioToken, obj, runtime, "audioToken");
343+
assignIfPresent(multimodalInputs.audios->token, obj, runtime, "audioToken");
344344
return multimodalInputs;
345345
}
346346

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,12 @@ std::string LLM::generateMultimodal(std::string prompt,
9999
std::vector<llm::MultimodalInput> inputs;
100100
size_t imageIdx = 0, audioIdx = 0, pos = 0;
101101
while (pos < prompt.size()) {
102-
size_t imgAt = mutlimodalInputs.imageToken.empty()
102+
size_t imgAt = mutlimodalInputs.images
103103
? std::string::npos
104-
: prompt.find(mutlimodalInputs.imageToken, pos);
105-
size_t audAt = mutlimodalInputs.audioToken.empty()
104+
: prompt.find(mutlimodalInputs.images->token, pos);
105+
size_t audAt = mutlimodalInputs.audios
106106
? std::string::npos
107-
: prompt.find(mutlimodalInputs.audioToken, pos);
107+
: prompt.find(mutlimodalInputs.audios->token, pos);
108108
if (imgAt == std::string::npos && audAt == std::string::npos) {
109109
inputs.push_back(llm::make_text_input(prompt.substr(pos)));
110110
break;
@@ -116,27 +116,27 @@ std::string LLM::generateMultimodal(std::string prompt,
116116
inputs.push_back(llm::make_text_input(prompt.substr(pos, at - pos)));
117117
}
118118
if (imageFirst) {
119-
if (imageIdx >= mutlimodalInputs.imagePaths.size()) {
119+
if (imageIdx >= mutlimodalInputs.images->paths.size()) {
120120
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
121-
"More '" + mutlimodalInputs.imageToken +
121+
"More '" + mutlimodalInputs.images->token +
122122
"' placeholders than image paths");
123123
}
124124
inputs.push_back(
125-
llm::make_image_input(mutlimodalInputs.imagePaths[imageIdx++]));
126-
pos = at + mutlimodalInputs.imageToken.size();
125+
llm::make_image_input(mutlimodalInputs.images->paths[imageIdx++]));
126+
pos = at + mutlimodalInputs.images->token.size();
127127
} else {
128-
if (audioIdx >= mutlimodalInputs.audioWaveforms.size()) {
128+
if (audioIdx >= mutlimodalInputs.audios->waveforms.size()) {
129129
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
130130
"More '" + mutlimodalInputs.audioToken +
131131
"' placeholders than audio waveforms");
132132
}
133133
inputs.push_back(llm::make_audio_input(
134-
std::move(mutlimodalInputs.audioWaveforms[audioIdx++])));
135-
pos = at + mutlimodalInputs.audioToken.size();
134+
std::move(mutlimodalInputs.audios->waveforms[audioIdx++])));
135+
pos = at + mutlimodalInputs.audios->token.size();
136136
}
137137
}
138-
if (imageIdx < mutlimodalInputs.imagePaths.size() ||
139-
audioIdx < mutlimodalInputs.audioWaveforms.size()) {
138+
if (imageIdx < mutlimodalInputs.images->paths.size() ||
139+
audioIdx < mutlimodalInputs.audios->waveforms.size()) {
140140
throw RnExecutorchError(
141141
RnExecutorchErrorCode::InvalidUserInput,
142142
"More image/audio paths provided than placeholders in prompt");

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.h‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,7 @@ class LLM : public BaseModel {
2323

2424
std::string generate(std::string prompt,
2525
std::shared_ptr<jsi::Function> callback);
26-
// Audio variant: `audioWaveforms` is a parallel vector of fp32 mono 16 kHz
27-
// PCM buffers (decoded upstream, same contract as SpeechToText::transcribe).
28-
// The prompt is scanned for `imageToken` and/or `audioToken` placeholders;
29-
// each placeholder consumes the next entry from its respective vector in
30-
// order. Either set of paths/waveforms/token may be empty.
26+
3127
std::string generateMultimodal(std::string prompt,
3228
std::shared_ptr<jsi::Function> callback,
3329
MultimodalInputs mutlimodalInputs = {});

‎packages/react-native-executorch/common/rnexecutorch/models/llm/Types.h‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,19 @@
44
#include <vector>
55

66
namespace rnexecutorch::models::llm {
7+
struct ImageInputs {
8+
std::vector<std::string> paths;
9+
std::string token;
10+
};
11+
12+
struct AudioInputs {
13+
std::vector<std::vector<float>> waveforms;
14+
std::string token;
15+
};
16+
717
struct MultimodalInputs {
8-
std::vector<std::string> imagePaths;
9-
std::string imageToken;
10-
std::vector<std::vector<float>> audioWaveforms;
11-
std::string audioToken;
18+
std::optional<ImageInputs> images;
19+
std::optional<AudioInputs> audios;
1220
};
13-
using Token = uint64_t;
1421

15-
} // namespace rnexecutorch::models::llm
22+
} // namespace rnexecutorch::models::llm

‎packages/react-native-executorch/common/runner/encoders/audio_encoder.cpp‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@ using ::executorch::runtime::EValue;
2020
using ::executorch::runtime::Result;
2121

2222
namespace {
23+
constexpr int32_t kSamplingRate = 16e3;
24+
constexpr int32_t kMaxLengthSeconds = 30;
2325
constexpr int32_t kSamplesPerBlock = 7680;
2426
constexpr int64_t kAudioBlockKMin = 1;
25-
constexpr int64_t kAudioBlockKMax = 62;
27+
constexpr int64_t kAudioBlockKMax =
28+
kSamplingRate * kMaxLengthSeconds / kSamplesPerBlock;
2629
} // namespace
2730

2831
AudioEncoder::AudioEncoder(::executorch::extension::Module &module)
@@ -70,15 +73,8 @@ Result<EValue> AudioEncoder::encode(const MultimodalInput &input) {
7073
ET_CHECK_OR_RETURN_ERROR(
7174
k_blocks >= kAudioBlockKMin && k_blocks <= kAudioBlockKMax,
7275
InvalidArgument,
73-
"AudioEncoder: waveform of %lld samples needs k_blocks=%lld; "
74-
"audio_encoder accepts k in [%lld, %lld] (block=%d samples; max %.2f s "
75-
"@ 16 kHz)",
76-
static_cast<long long>(n_valid), static_cast<long long>(k_blocks),
77-
static_cast<long long>(kAudioBlockKMin),
78-
static_cast<long long>(kAudioBlockKMax),
79-
static_cast<int>(kSamplesPerBlock),
80-
static_cast<double>(kSamplesPerBlock) *
81-
static_cast<double>(kAudioBlockKMax) / 16000.0);
76+
"AudioEncoder: waveform of %lld samples needs k_blocks=%lld.",
77+
static_cast<long long>(n_valid), static_cast<long long>(k_blocks));
8278
const int64_t n_padded = k_blocks * kSamplesPerBlock;
8379

8480
// Own the padded waveform for the lifetime of this call; from_blob below

0 commit comments

Comments
 (0)