Skip to content

Commit 7d145b5

Browse files
authored
feat: Adding support for gemma4 e2b models (#1162)
## Description <!-- Provide a concise and descriptive summary of the changes implemented in this PR. --> ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [x] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [x] Android ### Testing instructions Test by running apps/llm app on llm screen (for text only model) and multimodal screen (for audio-vision-text model). Text model should work as any other llm model. Multimodal can process up-to-30sec audio chunks as well as image inputs, should be able to transcribe audio, describe pictures or similar. ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues #1062 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [ ] My changes generate no new warnings ### Additional notes
1 parent 82fcf94 commit 7d145b5

46 files changed

Lines changed: 1729 additions & 351 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/llm/app/multimodal_llm/index.tsx‎

Lines changed: 250 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ import {
1212
View,
1313
} from 'react-native';
1414
import { launchImageLibrary } from 'react-native-image-picker';
15+
import {
16+
AudioManager,
17+
AudioRecorder,
18+
AudioContext,
19+
} from 'react-native-audio-api';
1520
import { useIsFocused } from '@react-navigation/native';
1621
import { useSafeAreaInsets } from 'react-native-safe-area-context';
1722
import { models, useLLM } from 'react-native-executorch';
@@ -23,12 +28,14 @@ import Spinner from '../../components/Spinner';
2328
import { GeneratingContext } from '../../context';
2429
import SuggestedPrompts from '../../components/SuggestedPrompts';
2530
import ErrorBanner from '../../components/ErrorBanner';
31+
import AudioWaveform from '../../components/AudioWaveform';
2632

2733
const SUGGESTED_PROMPTS = [
2834
"What's in this image?",
2935
'Describe this scene in detail',
3036
'What objects can you see?',
3137
'What text appears in this image?',
38+
'Transcribe the audio',
3239
];
3340
import { useLLMStats } from '../../hooks/useLLMStats';
3441
import { StatsBar } from '../../components/StatsBar';
@@ -46,12 +53,18 @@ function MultimodalLLMScreen() {
4653
const textInputRef = useRef<TextInput>(null);
4754
const { setGlobalGenerating } = useContext(GeneratingContext);
4855

49-
// Added error state
50-
const [error, setError] = useState<string | null>(null);
56+
const [audioBuffer, setAudioBuffer] = useState<Float32Array | null>(null);
57+
const [audioLabel, setAudioLabel] = useState<string | null>(null);
58+
const [audioUrl, setAudioUrl] = useState('');
59+
const [isFetchingAudio, setIsFetchingAudio] = useState(false);
60+
const [isRecording, setIsRecording] = useState(false);
61+
const [hasMicPermission, setHasMicPermission] = useState(false);
62+
const recorder = useRef(new AudioRecorder());
63+
const recordChunks = useRef<Float32Array[]>([]);
5164

52-
const vlm = useLLM({
53-
model: models.llm.lfm2_5_vl_1_6b(),
54-
});
65+
const [error, setError] = useState<string | null>(null);
66+
const model = models.llm.gemma4_e2b_multimodal();
67+
const vlm = useLLM({ model: model });
5568
const tokenCount = vlm.isReady ? vlm.getGeneratedTokenCount() : 0;
5669
const { stats, onMessageSend } = useLLMStats(
5770
vlm.response,
@@ -68,6 +81,95 @@ function MultimodalLLMScreen() {
6881
if (vlm.error) setError(String(vlm.error));
6982
}, [vlm.error]);
7083

84+
useEffect(() => {
85+
AudioManager.setAudioSessionOptions({
86+
iosCategory: 'playAndRecord',
87+
iosMode: 'spokenAudio',
88+
iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'],
89+
});
90+
(async () => {
91+
const status = await AudioManager.requestRecordingPermissions();
92+
setHasMicPermission(status === 'Granted');
93+
})();
94+
95+
return () => {
96+
if (vlm.isGenerating) vlm.interrupt();
97+
// eslint-disable-next-line react-hooks/exhaustive-deps
98+
recorder.current.stop();
99+
AudioManager.setAudioSessionActivity(false);
100+
};
101+
// eslint-disable-next-line react-hooks/exhaustive-deps
102+
}, []);
103+
104+
const loadAudioFromUrl = async () => {
105+
const url = audioUrl.trim();
106+
if (!url) return;
107+
setIsFetchingAudio(true);
108+
try {
109+
const ctx = new AudioContext({ sampleRate: 16000 });
110+
const decoded = await ctx.decodeAudioData(url);
111+
const pcm = decoded.getChannelData(0);
112+
const name = url.split('/').pop() || 'audio';
113+
setAudioBuffer(pcm);
114+
setAudioLabel(`${name} · ${(pcm.length / 16000).toFixed(1)}s`);
115+
} catch (e) {
116+
setError(e instanceof Error ? e.message : String(e));
117+
} finally {
118+
setIsFetchingAudio(false);
119+
}
120+
};
121+
122+
const startRecording = async () => {
123+
if (!hasMicPermission) {
124+
setError('Microphone permission denied. Please enable it in Settings.');
125+
return;
126+
}
127+
recordChunks.current = [];
128+
const sampleRate = 16000;
129+
recorder.current.onAudioReady(
130+
{ sampleRate, bufferLength: 0.1 * sampleRate, channelCount: 1 },
131+
({ buffer }) => {
132+
recordChunks.current.push(new Float32Array(buffer.getChannelData(0)));
133+
}
134+
);
135+
try {
136+
const ok = await AudioManager.setAudioSessionActivity(true);
137+
if (!ok) {
138+
setError('Cannot start audio session');
139+
return;
140+
}
141+
const result = recorder.current.start();
142+
if (result.status === 'error') {
143+
setError(`Recording problems: ${result.message}`);
144+
return;
145+
}
146+
setIsRecording(true);
147+
} catch (e) {
148+
setError(e instanceof Error ? e.message : String(e));
149+
}
150+
};
151+
152+
const stopRecording = () => {
153+
recorder.current.stop();
154+
setIsRecording(false);
155+
const total = recordChunks.current.reduce((n, c) => n + c.length, 0);
156+
if (total === 0) return;
157+
const pcm = new Float32Array(total);
158+
let off = 0;
159+
for (const c of recordChunks.current) {
160+
pcm.set(c, off);
161+
off += c.length;
162+
}
163+
recordChunks.current = [];
164+
setAudioBuffer(pcm);
165+
setAudioLabel(`Recording · ${(pcm.length / 16000).toFixed(1)}s`);
166+
};
167+
168+
const clearAudio = () => {
169+
setAudioBuffer(null);
170+
setAudioLabel(null);
171+
};
172+
71173
const pickImage = async () => {
72174
try {
73175
const result = await launchImageLibrary({ mediaType: 'photo' });
@@ -81,19 +183,27 @@ function MultimodalLLMScreen() {
81183
};
82184

83185
const sendMessage = async () => {
84-
if (!userInput.trim() || vlm.isGenerating) return;
186+
if (!(imageUri || audioBuffer || userInput.trim()) || vlm.isGenerating)
187+
return;
85188
onMessageSend();
86189
const text = userInput.trim();
87190
setUserInput('');
88191
textInputRef.current?.clear();
89192
Keyboard.dismiss();
90193
const currentImageUri = imageUri;
194+
const currentAudio = audioBuffer;
91195
setImageUri(null);
196+
setAudioBuffer(null);
197+
setAudioLabel(null);
92198
try {
93-
await vlm.sendMessage(
94-
text,
95-
currentImageUri ? { imagePath: currentImageUri } : undefined
96-
);
199+
const media =
200+
currentImageUri || currentAudio
201+
? {
202+
...(currentImageUri ? { imagePath: currentImageUri } : {}),
203+
...(currentAudio ? { audioBuffer: currentAudio } : {}),
204+
}
205+
: undefined;
206+
await vlm.sendMessage(text, media);
97207
} catch (e) {
98208
// Updated to set UI error instead of just console.error
99209
setError(e instanceof Error ? e.message : String(e));
@@ -135,7 +245,9 @@ function MultimodalLLMScreen() {
135245
<View style={styles.helloMessageContainer}>
136246
<Text style={styles.helloText}>Hello! 👋</Text>
137247
<Text style={styles.bottomHelloText}>
138-
Pick an image and ask me anything about it.
248+
{model.capabilities.find((c) => c === 'audio')
249+
? 'Say hi, or pick an image, and ask me anything about it.'
250+
: 'Pick an image and ask me anything about it.'}
139251
</Text>
140252
<SuggestedPrompts
141253
prompts={SUGGESTED_PROMPTS}
@@ -159,6 +271,48 @@ function MultimodalLLMScreen() {
159271
</TouchableOpacity>
160272
)}
161273

274+
{/* Audio URL input */}
275+
<View style={styles.audioUrlRow}>
276+
<TextInput
277+
placeholder="Audio URL (mp3/wav/…)"
278+
placeholderTextColor="#C1C6E5"
279+
style={styles.audioUrlInput}
280+
value={audioUrl}
281+
onChangeText={setAudioUrl}
282+
autoCapitalize="none"
283+
autoCorrect={false}
284+
/>
285+
<TouchableOpacity
286+
style={[
287+
styles.audioUrlButton,
288+
(!audioUrl.trim() || isFetchingAudio || vlm.isGenerating) &&
289+
styles.disabled,
290+
]}
291+
onPress={loadAudioFromUrl}
292+
disabled={!audioUrl.trim() || isFetchingAudio || vlm.isGenerating}
293+
>
294+
<Text style={styles.audioUrlButtonText}>
295+
{isFetchingAudio ? '…' : 'Load'}
296+
</Text>
297+
</TouchableOpacity>
298+
</View>
299+
300+
{/* Audio attachment strip */}
301+
{audioLabel && (
302+
<View style={styles.audioAttachmentContainer}>
303+
<View style={styles.audioAttachmentRow}>
304+
<Text style={styles.audioAttachmentText}>🎵 {audioLabel}</Text>
305+
<TouchableOpacity onPress={clearAudio}>
306+
<Text style={styles.audioAttachmentClear}>✕</Text>
307+
</TouchableOpacity>
308+
</View>
309+
<AudioWaveform
310+
buffer={audioBuffer}
311+
style={styles.audioWaveform}
312+
/>
313+
</View>
314+
)}
315+
162316
<StatsBar stats={stats} />
163317
<View
164318
style={[
@@ -178,6 +332,17 @@ function MultimodalLLMScreen() {
178332
<Text style={styles.imageButtonText}>📷</Text>
179333
</TouchableOpacity>
180334

335+
{/* Mic record / stop button */}
336+
<TouchableOpacity
337+
style={styles.imageButton}
338+
onPress={isRecording ? stopRecording : startRecording}
339+
disabled={vlm.isGenerating}
340+
>
341+
<Text style={styles.imageButtonText}>
342+
{isRecording ? '⏹️' : '🎤'}
343+
</Text>
344+
</TouchableOpacity>
345+
181346
<TextInput
182347
autoCorrect={false}
183348
ref={textInputRef}
@@ -198,14 +363,15 @@ function MultimodalLLMScreen() {
198363
onChangeText={setUserInput}
199364
/>
200365

201-
{userInput.trim() && !vlm.isGenerating && (
202-
<TouchableOpacity
203-
style={styles.sendChatTouchable}
204-
onPress={sendMessage}
205-
>
206-
<SendIcon height={24} width={24} padding={4} margin={8} />
207-
</TouchableOpacity>
208-
)}
366+
{(imageUri || audioBuffer || userInput.trim()) &&
367+
!vlm.isGenerating && (
368+
<TouchableOpacity
369+
style={styles.sendChatTouchable}
370+
onPress={sendMessage}
371+
>
372+
<SendIcon height={24} width={24} padding={4} margin={8} />
373+
</TouchableOpacity>
374+
)}
209375
{vlm.isGenerating && (
210376
<TouchableOpacity
211377
style={styles.sendChatTouchable}
@@ -319,6 +485,71 @@ const styles = StyleSheet.create({
319485
fontFamily: 'regular',
320486
color: ColorPalette.blueDark,
321487
},
488+
audioAttachmentContainer: {
489+
flexDirection: 'column',
490+
paddingHorizontal: 16,
491+
paddingVertical: 8,
492+
marginHorizontal: 16,
493+
marginBottom: 4,
494+
borderRadius: 8,
495+
borderWidth: 1,
496+
borderColor: ColorPalette.blueLight,
497+
backgroundColor: '#fafbff',
498+
},
499+
audioAttachmentRow: {
500+
flexDirection: 'row',
501+
alignItems: 'center',
502+
justifyContent: 'space-between',
503+
},
504+
audioAttachmentText: {
505+
fontSize: 13,
506+
fontFamily: 'regular',
507+
color: ColorPalette.blueDark,
508+
},
509+
audioAttachmentClear: {
510+
fontSize: 16,
511+
color: ColorPalette.blueDark,
512+
paddingHorizontal: 8,
513+
},
514+
audioWaveform: {
515+
marginTop: 6,
516+
minWidth: 0,
517+
},
518+
audioUrlRow: {
519+
flexDirection: 'row',
520+
alignItems: 'center',
521+
marginHorizontal: 16,
522+
marginBottom: 4,
523+
},
524+
audioUrlInput: {
525+
flex: 1,
526+
padding: 10,
527+
borderTopLeftRadius: 8,
528+
borderBottomLeftRadius: 8,
529+
borderWidth: 1,
530+
borderColor: ColorPalette.blueLight,
531+
borderRightWidth: 0,
532+
fontFamily: 'regular',
533+
fontSize: 13,
534+
color: ColorPalette.primary,
535+
},
536+
audioUrlButton: {
537+
paddingVertical: 10,
538+
paddingHorizontal: 16,
539+
backgroundColor: ColorPalette.strongPrimary,
540+
borderTopRightRadius: 8,
541+
borderBottomRightRadius: 8,
542+
justifyContent: 'center',
543+
alignItems: 'center',
544+
},
545+
audioUrlButtonText: {
546+
color: '#fff',
547+
fontFamily: 'medium',
548+
fontSize: 13,
549+
},
550+
disabled: {
551+
opacity: 0.5,
552+
},
322553
bottomContainer: {
323554
height: 100,
324555
width: '100%',

0 commit comments

Comments
 (0)