Skip to content

Commit 27776de

Browse files
mkopcinsmkopcins
committed
gemma_vulkan
1 parent 668c37c commit 27776de

29 files changed

Lines changed: 811 additions & 288 deletions

‎apps/llm/Screen_Recording_20260511_112819_llm.mp4‎

4.93 MB
Binary file not shown.

‎apps/llm/app/llm/index.tsx‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,11 @@ function LLMScreen() {
4242
const { bottom } = useSafeAreaInsets();
4343
const [isTextInputFocused, setIsTextInputFocused] = useState(false);
4444
const [userInput, setUserInput] = useState('');
45-
const [selectedModel, setSelectedModel] = useState<LLMModelSources>(
46-
models.llm.lfm2_5_1_2b_instruct()
47-
);
4845
const textInputRef = useRef<TextInput>(null);
4946
const { setGlobalGenerating } = useContext(GeneratingContext);
5047

51-
const llm = useLLM({ model: selectedModel });
48+
let llm = Object();
49+
5250
const tokenCount = llm.isReady ? llm.getGeneratedTokenCount() : 0;
5351
const { stats, onMessageSend } = useLLMStats(
5452
llm.response,
@@ -112,13 +110,6 @@ function LLMScreen() {
112110
/>
113111
</View>
114112
)}
115-
116-
<ModelPicker
117-
models={LLM_MODELS}
118-
selectedModel={selectedModel}
119-
onSelect={(m) => setSelectedModel(m)}
120-
disabled={llm.isGenerating}
121-
/>
122113
<StatsBar stats={stats} />
123114
<View
124115
style={[

‎apps/llm/app/multimodal_llm/index.tsx‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ const SUGGESTED_PROMPTS = [
3434
'Describe this scene in detail',
3535
'What objects can you see?',
3636
'What text appears in this image?',
37+
'Transcribe the audio?',
3738
];
3839
import { useLLMStats } from '../../hooks/useLLMStats';
3940
import { StatsBar } from '../../components/StatsBar';
@@ -63,7 +64,7 @@ function MultimodalLLMScreen() {
6364
const [error, setError] = useState<string | null>(null);
6465

6566
const vlm = useLLM({
66-
model: models.llm.lfm2_5_vl_1_6b(),
67+
model: models.llm.gemma4_e2b(),
6768
});
6869
const tokenCount = vlm.isReady ? vlm.getGeneratedTokenCount() : 0;
6970
const { stats, onMessageSend } = useLLMStats(
@@ -175,7 +176,8 @@ function MultimodalLLMScreen() {
175176
};
176177

177178
const sendMessage = async () => {
178-
if (!userInput.trim() || vlm.isGenerating) return;
179+
if (!(imageUri || audioBuffer || userInput.trim()) || vlm.isGenerating)
180+
return;
179181
onMessageSend();
180182
const text = userInput.trim();
181183
setUserInput('');
@@ -346,14 +348,15 @@ function MultimodalLLMScreen() {
346348
onChangeText={setUserInput}
347349
/>
348350

349-
{userInput.trim() && !vlm.isGenerating && (
350-
<TouchableOpacity
351-
style={styles.sendChatTouchable}
352-
onPress={sendMessage}
353-
>
354-
<SendIcon height={24} width={24} padding={4} margin={8} />
355-
</TouchableOpacity>
356-
)}
351+
{(imageUri || audioBuffer || userInput.trim()) &&
352+
!vlm.isGenerating && (
353+
<TouchableOpacity
354+
style={styles.sendChatTouchable}
355+
onPress={sendMessage}
356+
>
357+
<SendIcon height={24} width={24} padding={4} margin={8} />
358+
</TouchableOpacity>
359+
)}
357360
{vlm.isGenerating && (
358361
<TouchableOpacity
359362
style={styles.sendChatTouchable}

‎packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -223,8 +223,6 @@ inline std::vector<float> getValue<std::vector<float>>(const jsi::Value &val,
223223
return getArrayAsVector<float>(val, runtime);
224224
}
225225

226-
// JS side passes an Array<Float32Array> (one clip per element). Each inner
227-
// element is read as a typed-array span and copied into a std::vector<float>.
228226
template <>
229227
inline std::vector<std::vector<float>>
230228
getValue<std::vector<std::vector<float>>>(const jsi::Value &val,

‎packages/react-native-executorch/common/rnexecutorch/host_objects/ModelHostObject.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -195,11 +195,6 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
195195
promiseHostFunction<&Model::generateMultimodal>,
196196
"generateMultimodal"));
197197

198-
addFunctions(JSI_EXPORT_FUNCTION(
199-
ModelHostObject<Model>,
200-
promiseHostFunction<&Model::generateMultimodalWithAudio>,
201-
"generateMultimodalWithAudio"));
202-
203198
addFunctions(JSI_EXPORT_FUNCTION(
204199
ModelHostObject<Model>,
205200
synchronousHostFunction<&Model::getVisualTokenCount>,

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.cpp‎

Lines changed: 5 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#include <filesystem>
55
#include <map>
66
#include <rnexecutorch/Error.h>
7-
#include <rnexecutorch/Log.h>
87
#include <rnexecutorch/threads/GlobalThreadPool.h>
98
#include <runner/encoders/audio_encoder.h>
109
#include <runner/encoders/vision_encoder.h>
@@ -22,7 +21,6 @@ LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource,
2221
std::vector<std::string> capabilities,
2322
std::shared_ptr<react::CallInvoker> callInvoker)
2423
: BaseModel(modelSource, callInvoker, Module::LoadMode::Mmap) {
25-
2624
if (capabilities.empty()) {
2725
runner_ =
2826
std::make_unique<llm::TextRunner>(std::move(module_), tokenizerSource);
@@ -78,90 +76,10 @@ std::string LLM::generate(std::string input,
7876
return output;
7977
}
8078

81-
std::string LLM::generateMultimodal(std::string prompt,
82-
std::vector<std::string> imagePaths,
83-
std::string imageToken,
84-
std::shared_ptr<jsi::Function> callback) {
85-
if (!runner_ || !runner_->is_loaded()) {
86-
throw RnExecutorchError(RnExecutorchErrorCode::ModuleNotLoaded,
87-
"Runner is not loaded");
88-
}
89-
if (!runner_->is_multimodal()) {
90-
throw RnExecutorchError(
91-
RnExecutorchErrorCode::InvalidUserInput,
92-
"This model does not support multimodal input. Use generate(prompt, "
93-
"callback) for text-only generation.");
94-
}
95-
if (imageToken.empty()) {
96-
throw RnExecutorchError(
97-
RnExecutorchErrorCode::InvalidUserInput,
98-
"imageToken must not be empty. Pass the model's image token (e.g. "
99-
"from tokenizer_config.json).");
100-
}
101-
102-
const size_t kImageTokenLen = imageToken.size();
103-
104-
std::vector<llm::MultimodalInput> inputs;
105-
size_t imageIdx = 0;
106-
size_t searchPos = 0;
107-
108-
while (true) {
109-
size_t found = prompt.find(imageToken, searchPos);
110-
if (found == std::string::npos) {
111-
if (searchPos < prompt.size()) {
112-
inputs.push_back(llm::make_text_input(prompt.substr(searchPos)));
113-
}
114-
break;
115-
}
116-
// Text segment before this placeholder
117-
if (found > searchPos) {
118-
inputs.push_back(
119-
llm::make_text_input(prompt.substr(searchPos, found - searchPos)));
120-
}
121-
// Image at this position
122-
if (imageIdx >= imagePaths.size()) {
123-
throw RnExecutorchError(
124-
RnExecutorchErrorCode::InvalidUserInput,
125-
"More '" + imageToken +
126-
"' placeholders in prompt than image paths provided");
127-
}
128-
inputs.push_back(llm::make_image_input(imagePaths[imageIdx++]));
129-
searchPos = found + kImageTokenLen;
130-
}
131-
132-
if (imageIdx < imagePaths.size()) {
133-
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
134-
"More image paths provided than '" + imageToken +
135-
"' placeholders in prompt");
136-
}
137-
138-
if (inputs.empty()) {
139-
throw RnExecutorchError(RnExecutorchErrorCode::InvalidUserInput,
140-
"No inputs to generate from");
141-
}
142-
143-
std::string output;
144-
auto nativeCallback = [this, callback, &output](const std::string &token) {
145-
output += token;
146-
if (callback && callInvoker) {
147-
callInvoker->invokeAsync([callback, token](jsi::Runtime &runtime) {
148-
callback->call(runtime, jsi::String::createFromUtf8(runtime, token));
149-
});
150-
}
151-
};
152-
153-
auto error = runner_->generate(inputs, nativeCallback);
154-
if (error != Error::Ok) {
155-
throw RnExecutorchError(error, "Failed to generate multimodal response");
156-
}
157-
158-
return output;
159-
}
160-
161-
std::string LLM::generateMultimodalWithAudio(
162-
std::string prompt, std::vector<std::string> imagePaths,
163-
std::string imageToken, std::vector<std::vector<float>> audioWaveforms,
164-
std::string audioToken, std::shared_ptr<jsi::Function> callback) {
79+
std::string LLM::generateMultimodal(
80+
std::string prompt, std::shared_ptr<jsi::Function> callback,
81+
std::vector<std::string> imagePaths, std::string imageToken,
82+
std::vector<std::vector<float>> audioWaveforms, std::string audioToken) {
16583
if (!runner_ || !runner_->is_loaded()) {
16684
throw RnExecutorchError(RnExecutorchErrorCode::ModuleNotLoaded,
16785
"Runner is not loaded");
@@ -234,6 +152,7 @@ std::string LLM::generateMultimodalWithAudio(
234152
});
235153
}
236154
};
155+
237156
auto error = runner_->generate(inputs, nativeCallback);
238157
if (error != Error::Ok) {
239158
throw RnExecutorchError(error, "Failed to generate multimodal response");

‎packages/react-native-executorch/common/rnexecutorch/models/llm/LLM.h‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,16 @@ class LLM : public BaseModel {
2222

2323
std::string generate(std::string prompt,
2424
std::shared_ptr<jsi::Function> callback);
25-
std::string generateMultimodal(std::string prompt,
26-
std::vector<std::string> imagePaths,
27-
std::string imageToken,
28-
std::shared_ptr<jsi::Function> callback);
2925
// Audio variant: `audioWaveforms` is a parallel vector of fp32 mono 16 kHz
3026
// PCM buffers (decoded upstream, same contract as SpeechToText::transcribe).
3127
// The prompt is scanned for `imageToken` and/or `audioToken` placeholders;
3228
// each placeholder consumes the next entry from its respective vector in
3329
// order. Either set of paths/waveforms/token may be empty.
34-
std::string generateMultimodalWithAudio(
35-
std::string prompt, std::vector<std::string> imagePaths,
36-
std::string imageToken, std::vector<std::vector<float>> audioWaveforms,
37-
std::string audioToken, std::shared_ptr<jsi::Function> callback);
30+
std::string generateMultimodal(
31+
std::string prompt, std::shared_ptr<jsi::Function> callback,
32+
std::vector<std::string> imagePaths = {}, std::string imageToken = "",
33+
std::vector<std::vector<float>> audioWaveforms = {},
34+
std::string audioToken = "");
3835

3936
void interrupt();
4037
void reset();

‎packages/react-native-executorch/common/runner/base_llm_runner.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,8 @@ void BaseLLMRunner::set_repetition_penalty(float repetition_penalty) noexcept {
149149
config_.repetition_penalty = repetition_penalty;
150150
}
151151

152+
void BaseLLMRunner::set_topk(int32_t topk) noexcept { config_.topk = topk; }
153+
152154
void BaseLLMRunner::set_count_interval(size_t count_interval) {
153155
config_.output_token_batch_size = count_interval;
154156
}

‎packages/react-native-executorch/common/runner/base_llm_runner.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ class BaseLLMRunner {
5555
void set_topp(float topp) noexcept;
5656
void set_min_p(float min_p) noexcept;
5757
void set_repetition_penalty(float repetition_penalty) noexcept;
58+
void set_topk(int32_t topk) noexcept;
5859
void set_count_interval(size_t count_interval);
5960
void set_time_interval(size_t time_interval);
6061

‎packages/react-native-executorch/common/runner/constants.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,11 @@ inline constexpr auto kVisionEncoderMethod = "vision_encoder";
2323
inline constexpr auto kAudioEncoderMethod = "audio_encoder";
2424
inline constexpr auto kTokenEmbeddingMethod = "token_embedding";
2525
inline constexpr auto kTextModelMethod = "text_decoder";
26-
inline constexpr auto kMaxPrefillLen = 1024;
26+
// Absolute ceiling on prefill length (in tokens) and the fallback value used
27+
// when a PTE doesn't bake `get_max_seq_len`. 2048 matches Gemma4 iter201's
28+
// PREFILL_LEN / get_max_context_len; legacy PTEs (e.g. LFM2-VL) typically
29+
// bake their own get_max_seq_len so this ceiling does not affect them.
30+
inline constexpr auto kMaxPrefillLen = 2048;
2731
inline constexpr auto numOfAddedBoSTokens = 0;
2832
inline constexpr auto numOfAddedEoSTokens = 0;
2933

0 commit comments

Comments
 (0)