Skip to content

Commit 71ba63c

Browse files
mkopcinsmkopcins
committed
review fix
1 parent 539a125 commit 71ba63c

10 files changed

Lines changed: 124 additions & 90 deletions

File tree

‎apps/llm/app/multimodal_llm/index.tsx‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,8 @@ function MultimodalLLMScreen() {
6363
const recordChunks = useRef<Float32Array[]>([]);
6464

6565
const [error, setError] = useState<string | null>(null);
66-
67-
const vlm = useLLM({
68-
model: models.llm.gemma4_e2b_multimodal(),
69-
});
66+
const model = models.llm.gemma4_e2b_multimodal();
67+
const vlm = useLLM({ model: model });
7068
const tokenCount = vlm.isReady ? vlm.getGeneratedTokenCount() : 0;
7169
const { stats, onMessageSend } = useLLMStats(
7270
vlm.response,
@@ -93,6 +91,12 @@ function MultimodalLLMScreen() {
9391
const status = await AudioManager.requestRecordingPermissions();
9492
setHasMicPermission(status === 'Granted');
9593
})();
94+
95+
return () => {
96+
// eslint-disable-next-line react-hooks/exhaustive-deps
97+
recorder.current.stop();
98+
AudioManager.setAudioSessionActivity(false);
99+
};
96100
}, []);
97101

98102
const loadAudioFromUrl = async () => {
@@ -239,7 +243,9 @@ function MultimodalLLMScreen() {
239243
<View style={styles.helloMessageContainer}>
240244
<Text style={styles.helloText}>Hello! 👋</Text>
241245
<Text style={styles.bottomHelloText}>
242-
Pick an image and ask me anything about it.
246+
{model.capabilities.find((c) => c === 'audio')
247+
? 'Say hi or pick an image or and ask me anything about it.'
248+
: 'Pick an image and ask me anything about it.'}
243249
</Text>
244250
<SuggestedPrompts
245251
prompts={SUGGESTED_PROMPTS}

‎packages/react-native-executorch/common/rnexecutorch/models/llm/Types.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include <optional>
34
#include <string>
45
#include <vector>
56

‎packages/react-native-executorch/common/runner/encoders/audio_encoder.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,14 @@ Result<EValue> AudioEncoder::encode(const MultimodalInput &input) {
8787
std::memcpy(padded_wav_.data(), wav.samples.data(),
8888
static_cast<size_t>(n_valid) * sizeof(float));
8989

90-
num_blocks_scalar_ = n_valid;
90+
valid_samples_scalar_ = n_valid;
9191

9292
auto wav_tensor = ::executorch::extension::from_blob(
9393
padded_wav_.data(), {1, static_cast<SizesType>(n_padded)},
9494
::executorch::aten::ScalarType::Float);
9595

9696
auto num_blocks_tensor = ::executorch::extension::from_blob(
97-
&num_blocks_scalar_, {}, ::executorch::aten::ScalarType::Long);
97+
&valid_samples_scalar_, {}, ::executorch::aten::ScalarType::Long);
9898

9999
std::vector<EValue> args = {EValue(*wav_tensor), EValue(*num_blocks_tensor)};
100100
auto exec_result = ET_UNWRAP(module_->execute(kAudioEncoderMethod, args));

‎packages/react-native-executorch/common/runner/encoders/audio_encoder.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class AudioEncoder : public IEncoder {
3434
::executorch::extension::Module *module_;
3535
int32_t last_token_count_ = 0;
3636
std::vector<float> padded_wav_;
37-
int64_t num_blocks_scalar_ = 0;
37+
int64_t valid_samples_scalar_ = 0;
3838
};
3939

4040
} // namespace executorch::extension::llm

‎packages/react-native-executorch/common/runner/multimodal_prefiller.cpp‎

Lines changed: 63 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -26,47 +26,50 @@ using ::executorch::runtime::Result;
2626

2727
MultimodalPrefiller::MultimodalPrefiller(
2828
Module &module, MultimodalDecoderRunner &decoder_runner,
29-
tokenizers::HFTokenizer &tokenizer, IEncoder *image_encoder,
29+
tokenizers::HFTokenizer &tokenizer,
30+
std::unordered_map<std::string, int64_t> metadata, IEncoder *image_encoder,
3031
IEncoder *audio_encoder)
3132
: module_(&module), decoder_runner_(&decoder_runner),
32-
tokenizer_(&tokenizer), image_encoder_(image_encoder),
33-
audio_encoder_(audio_encoder) {}
33+
tokenizer_(&tokenizer), metadata_(metadata),
34+
image_encoder_(image_encoder), audio_encoder_(audio_encoder) {}
3435

3536
std::optional<int64_t> MultimodalPrefiller::get_max_seq_len() const {
3637
auto r = module_->get(kMaxSeqLen);
3738
if (r.error() != ::executorch::runtime::Error::Ok) {
38-
return std::nullopt;
39+
return metadata_.at(kMaxSeqLen);
3940
}
4041
return r->toScalar().to<int64_t>();
4142
}
4243

4344
std::optional<int64_t> MultimodalPrefiller::get_max_context_len() const {
4445
auto r = module_->get(kMaxContextLen);
4546
if (r.error() != ::executorch::runtime::Error::Ok) {
46-
return std::nullopt;
47+
return metadata_.at(kMaxContextLen);
4748
}
4849
return r->toScalar().to<int64_t>();
4950
}
5051

5152
bool MultimodalPrefiller::get_enable_dynamic_shape() const {
5253
auto r = module_->get(kEnableDynamicShape);
5354
if (r.error() != ::executorch::runtime::Error::Ok) {
54-
return false;
55+
return metadata_.at(kEnableDynamicShape);
5556
}
5657
return r->toScalar().to<bool>();
5758
}
5859

59-
auto MultimodalPrefiller::processMultimodalInput(
60+
[[nodiscard]] auto MultimodalPrefiller::processMultimodalInput(
6061
const MultimodalInput &input, std::vector<int64_t> &ids,
61-
std::vector<ImageSlot> &image_slots, std::vector<AudioSlot> &audio_slots) {
62+
std::vector<Types::ImageSlot> &image_slots,
63+
std::vector<Types::AudioSlot> &audio_slots) {
6264
if (input.is_image()) {
6365
ET_CHECK_OR_RETURN_ERROR(image_encoder_ != nullptr, InvalidState,
6466
"No image encoder registered");
6567
const int32_t num_visual = image_encoder_->encoderTokenCount();
6668
ET_CHECK_OR_RETURN_ERROR(num_visual > 0, InvalidState,
6769
"Image encoder reports 0 visual tokens");
68-
image_slots.push_back(ImageSlot{&input, static_cast<int64_t>(ids.size()),
69-
static_cast<int64_t>(num_visual)});
70+
image_slots.push_back(Types::ImageSlot{&input,
71+
static_cast<int64_t>(ids.size()),
72+
static_cast<int64_t>(num_visual)});
7073
ids.insert(ids.end(), static_cast<size_t>(num_visual), 0);
7174
} else if (input.is_audio()) {
7275
ET_CHECK_OR_RETURN_ERROR(audio_encoder_ != nullptr, InvalidState,
@@ -89,9 +92,9 @@ auto MultimodalPrefiller::processMultimodalInput(
8992
std::vector<uint8_t> bytes(audio_tensor.nbytes());
9093
std::memcpy(bytes.data(), audio_tensor.const_data_ptr(),
9194
audio_tensor.nbytes());
92-
audio_slots.push_back(
93-
AudioSlot{std::move(bytes), audio_tensor.scalar_type(),
94-
static_cast<int64_t>(ids.size()), num_audio, audio_hidden});
95+
audio_slots.push_back(Types::AudioSlot{
96+
std::move(bytes), audio_tensor.scalar_type(),
97+
static_cast<int64_t>(ids.size()), num_audio, audio_hidden});
9598
ids.insert(ids.end(), static_cast<size_t>(num_audio), 0);
9699
} else if (input.is_text()) {
97100
auto encode_result = tokenizer_->encode(input.get_text());
@@ -109,14 +112,16 @@ auto MultimodalPrefiller::processMultimodalInput(
109112
for (auto t : tokens) {
110113
ids.push_back(static_cast<int64_t>(t));
111114
}
115+
} else {
116+
ET_LOG(Error, "Unsupported MultimodalInput type");
117+
return Error::NotSupported;
112118
}
113-
ET_LOG(Error, "Unsupported MultimodalInput type");
114-
return Error::NotSupported;
119+
return ::executorch::runtime::Error::Ok;
115120
}
116121

117-
auto MultimodalPrefiller::encodeAudio(
118-
const AudioSlot &slot, const auto hidden, std::vector<uint8_t> &embeds_buf,
119-
const size_t embeds_elem_size,
122+
[[nodiscard]] auto MultimodalPrefiller::encodeAudio(
123+
const Types::AudioSlot &slot, const auto hidden,
124+
std::vector<uint8_t> &embeds_buf, const size_t embeds_elem_size,
120125
const ::executorch::aten::ScalarType &embeds_dtype) {
121126
ET_CHECK_OR_RETURN_ERROR(
122127
slot.audio_hidden == static_cast<int64_t>(hidden), InvalidState,
@@ -166,9 +171,9 @@ auto MultimodalPrefiller::encodeAudio(
166171
return ::executorch::runtime::Error::Ok;
167172
}
168173

169-
auto MultimodalPrefiller::encodeImages(
170-
const ImageSlot &slot, const auto hidden, std::vector<uint8_t> &embeds_buf,
171-
const size_t embeds_elem_size,
174+
[[nodiscard]] auto MultimodalPrefiller::encodeImages(
175+
const Types::ImageSlot &slot, const auto hidden,
176+
std::vector<uint8_t> &embeds_buf, const size_t embeds_elem_size,
172177
const ::executorch::aten::ScalarType &embeds_dtype) {
173178
auto encode_result = image_encoder_->encode(*slot.input);
174179
ET_CHECK_OK_OR_RETURN_ERROR(encode_result.error(), "Image encoding failed");
@@ -208,8 +213,9 @@ auto MultimodalPrefiller::encodeImages(
208213
return ::executorch::runtime::Error::Ok;
209214
}
210215

211-
auto MultimodalPrefiller::initializePLE(auto &embed_outputs, auto total_len,
212-
PLEEmbeddings &ple_embeddings) {
216+
[[nodiscard]] auto
217+
MultimodalPrefiller::initializePLE(auto &embed_outputs, auto total_len,
218+
Types::PLEEmbeddings &ple_embeddings) {
213219
auto full_ple_tok = embed_outputs[1].toTensor();
214220
ple_embeddings.num_layers = static_cast<SizesType>(full_ple_tok.size(2));
215221
ple_embeddings.ple_dim = static_cast<SizesType>(full_ple_tok.size(3));
@@ -218,24 +224,22 @@ auto MultimodalPrefiller::initializePLE(auto &embed_outputs, auto total_len,
218224
const size_t total_bytes = full_ple_tok.nbytes();
219225
ET_CHECK_OR_RETURN_ERROR(total_numel > 0, InvalidState,
220226
"ple_tok has zero elements");
221-
size_t ple_elem_size = total_bytes / total_numel;
227+
ple_embeddings.ple_elem_size = total_bytes / total_numel;
222228
const size_t prefix_bytes = static_cast<size_t>(total_len) *
223229
static_cast<size_t>(ple_embeddings.num_layers) *
224230
static_cast<size_t>(ple_embeddings.ple_dim) *
225-
ple_elem_size;
231+
ple_embeddings.ple_elem_size;
226232
ple_embeddings.ple_tok_buf.resize(prefix_bytes);
227233
std::memcpy(ple_embeddings.ple_tok_buf.data(),
228234
full_ple_tok.mutable_data_ptr(), prefix_bytes);
229235
return ::executorch::runtime::Error::Ok;
230236
}
231237

232-
auto MultimodalPrefiller::prefillChunk(std::vector<EValue> &last_outs,
233-
std::vector<uint8_t> &embeds_buf,
234-
auto chunk_start, auto chunk_len,
235-
auto hidden, auto embeds_elem_size,
236-
auto embeds_dtype,
237-
PLEEmbeddings &ple_embeddings,
238-
std::vector<int64_t> &cache_positions) {
238+
[[nodiscard]] auto MultimodalPrefiller::prefillChunk(
239+
std::vector<EValue> &last_outs, std::vector<uint8_t> &embeds_buf,
240+
auto chunk_start, auto chunk_len, auto hidden, auto embeds_elem_size,
241+
auto embeds_dtype, Types::PLEEmbeddings &ple_embeddings,
242+
std::vector<int64_t> &cache_positions) {
239243
uint8_t *embeds_chunk_ptr =
240244
embeds_buf.data() + static_cast<size_t>(chunk_start) *
241245
static_cast<size_t>(hidden) * embeds_elem_size;
@@ -296,11 +300,14 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
296300

297301
std::vector<int64_t> ids;
298302
ids.reserve(static_cast<size_t>(prefill_total_cap));
299-
std::vector<ImageSlot> image_slots;
300-
std::vector<AudioSlot> audio_slots;
303+
std::vector<Types::ImageSlot> image_slots;
304+
std::vector<Types::AudioSlot> audio_slots;
301305

302306
for (const auto &input : inputs) {
303-
processMultimodalInput(input, ids, image_slots, audio_slots);
307+
auto res = processMultimodalInput(input, ids, image_slots, audio_slots);
308+
if (res != ::executorch::runtime::Error::Ok) {
309+
return res;
310+
}
304311
}
305312

306313
const int64_t total_len = static_cast<int64_t>(ids.size());
@@ -350,7 +357,11 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
350357
// Pass 2: encode images and splice their outputs into embeds_buf.
351358
// ------------------------------------------------------------
352359
for (const auto &slot : image_slots) {
353-
encodeImages(slot, hidden, embeds_buf, embeds_elem_size, embeds_dtype);
360+
auto res =
361+
encodeImages(slot, hidden, embeds_buf, embeds_elem_size, embeds_dtype);
362+
if (res != ::executorch::runtime::Error::Ok) {
363+
return res;
364+
}
354365
}
355366

356367
// ------------------------------------------------------------
@@ -359,12 +370,19 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
359370
// invalidate slot state. Same dtype-conversion matrix as vision.
360371
// ------------------------------------------------------------
361372
for (auto &slot : audio_slots) {
362-
encodeAudio(slot, hidden, embeds_buf, embeds_elem_size, embeds_dtype);
373+
auto res =
374+
encodeAudio(slot, hidden, embeds_buf, embeds_elem_size, embeds_dtype);
375+
if (res != ::executorch::runtime::Error::Ok) {
376+
return res;
377+
}
363378
}
364379

365-
PLEEmbeddings ple_embeddings;
380+
Types::PLEEmbeddings ple_embeddings;
366381
if (has_ple) {
367-
initializePLE(embed_outputs, total_len, ple_embeddings);
382+
auto res = initializePLE(embed_outputs, total_len, ple_embeddings);
383+
if (res != ::executorch::runtime::Error::Ok) {
384+
return res;
385+
}
368386
}
369387

370388
std::vector<EValue> last_outs;
@@ -379,9 +397,12 @@ MultimodalPrefiller::prefill(const std::vector<MultimodalInput> &inputs,
379397
const int64_t chunk_start = ci * chunk_cap;
380398
const int64_t chunk_end = std::min(chunk_start + chunk_cap, total_len);
381399
const int64_t chunk_len = chunk_end - chunk_start;
382-
prefillChunk(last_outs, embeds_buf, chunk_start, chunk_len, hidden,
383-
embeds_elem_size, embeds_dtype, ple_embeddings,
384-
cache_positions);
400+
auto res = prefillChunk(last_outs, embeds_buf, chunk_start, chunk_len,
401+
hidden, embeds_elem_size, embeds_dtype,
402+
ple_embeddings, cache_positions);
403+
if (res != ::executorch::runtime::Error::Ok) {
404+
return res;
405+
}
385406
}
386407

387408
ET_CHECK_OR_RETURN_ERROR(!last_outs.empty(), InvalidState,

‎packages/react-native-executorch/common/runner/multimodal_prefiller.h‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
namespace executorch::extension::llm {
2020

21-
namespace {
21+
namespace Types {
2222
struct ImageSlot {
2323
const MultimodalInput *input; // non-owning, valid for duration of call
2424
int64_t slot_start;
@@ -38,17 +38,18 @@ struct PLEEmbeddings {
3838
aten::SizesType num_layers = 0;
3939
aten::SizesType ple_dim = 0;
4040
size_t ple_elem_size = 0;
41-
::executorch::aten::ScalarType ple_tok_dtype;
41+
::executorch::aten::ScalarType ple_tok_dtype =
42+
::executorch::aten::ScalarType::Half;
4243
};
43-
} // namespace
44+
} // namespace Types
4445

4546
class MultimodalPrefiller {
4647
public:
47-
explicit MultimodalPrefiller(Module &module,
48-
MultimodalDecoderRunner &decoder_runner,
49-
tokenizers::HFTokenizer &tokenizer,
50-
IEncoder *image_encoder = nullptr,
51-
IEncoder *audio_encoder = nullptr);
48+
explicit MultimodalPrefiller(
49+
Module &module, MultimodalDecoderRunner &decoder_runner,
50+
tokenizers::HFTokenizer &tokenizer,
51+
std::unordered_map<std::string, int64_t> metadata,
52+
IEncoder *image_encoder = nullptr, IEncoder *audio_encoder = nullptr);
5253

5354
// Prefill one input segment. Updates start_pos in-place.
5455
// Returns the first predicted token after this segment.
@@ -57,33 +58,35 @@ class MultimodalPrefiller {
5758

5859
auto processMultimodalInput(const MultimodalInput &input,
5960
std::vector<int64_t> &ids,
60-
std::vector<ImageSlot> &image_slots,
61-
std::vector<AudioSlot> &audio_slots);
62-
auto encodeImages(const ImageSlot &slot, const auto hidden,
61+
std::vector<Types::ImageSlot> &image_slots,
62+
std::vector<Types::AudioSlot> &audio_slots);
63+
::executorch::runtime::Error load();
64+
bool is_method_loaded();
65+
std::optional<int64_t> get_max_seq_len() const;
66+
std::optional<int64_t> get_max_context_len() const;
67+
bool get_enable_dynamic_shape() const;
68+
69+
private:
70+
auto encodeImages(const Types::ImageSlot &slot, const auto hidden,
6371
std::vector<uint8_t> &embeds_buf,
6472
const size_t embeds_elem_size,
6573
const ::executorch::aten::ScalarType &embeds_dtype);
66-
auto encodeAudio(const AudioSlot &slot, const auto hidden,
74+
auto encodeAudio(const Types::AudioSlot &slot, const auto hidden,
6775
std::vector<uint8_t> &embeds_buf,
6876
const size_t embeds_elem_size,
6977
const ::executorch::aten::ScalarType &embeds_dtype);
7078
auto prefillChunk(std::vector<::executorch::runtime::EValue> &last_outs,
7179
std::vector<uint8_t> &embeds_buf, auto chunk_start,
7280
auto chunk_len, auto hidden, auto embeds_elem_size,
73-
auto embeds_dtype, PLEEmbeddings &ple_embeddings,
81+
auto embeds_dtype, Types::PLEEmbeddings &ple_embeddings,
7482
std::vector<int64_t> &cache_positions);
7583
auto initializePLE(auto &embed_outputs, auto total_len,
76-
PLEEmbeddings &ple_embeddings);
77-
::executorch::runtime::Error load();
78-
bool is_method_loaded();
79-
std::optional<int64_t> get_max_seq_len() const;
80-
std::optional<int64_t> get_max_context_len() const;
81-
bool get_enable_dynamic_shape() const;
84+
Types::PLEEmbeddings &ple_embeddings);
8285

83-
private:
8486
Module *module_;
8587
MultimodalDecoderRunner *decoder_runner_;
8688
tokenizers::HFTokenizer *tokenizer_;
89+
std::unordered_map<std::string, int64_t> metadata_;
8790
IEncoder *image_encoder_;
8891
IEncoder *audio_encoder_;
8992
};

‎packages/react-native-executorch/common/runner/multimodal_runner.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ Error MultimodalRunner::load_subcomponents() {
5959
audio_encoder = aud_it->second.get();
6060
}
6161
mm_prefiller_ = std::make_unique<MultimodalPrefiller>(
62-
*module_, *mm_decoder_runner_, *tokenizer_, image_encoder, audio_encoder);
62+
*module_, *mm_decoder_runner_, *tokenizer_, metadata_, image_encoder,
63+
audio_encoder);
6364
mm_token_generator_ = std::make_unique<TextTokenGenerator>(
6465
tokenizer_.get(), mm_decoder_runner_.get(), /*use_kv_cache=*/true,
6566
std::move(eos_ids_), stats_ptr, config_);
@@ -98,8 +99,8 @@ Error MultimodalRunner::generate_internal(
9899
? config_.max_context_length
99100
: config_.max_seq_len;
100101
int32_t resolved_max_new = resolve_max_new_tokens(
101-
static_cast<int32_t>(pos_), seq_cap,
102-
config_.max_context_length, config_.max_new_tokens);
102+
static_cast<int32_t>(pos_), seq_cap, config_.max_context_length,
103+
config_.max_new_tokens);
103104

104105
std::vector<uint64_t> seed_tokens = {prefill_next_token};
105106
auto wrapped_callback = [&](const std::string &piece) {

0 commit comments

Comments
 (0)