@@ -12,6 +12,11 @@ import {
1212 View ,
1313} from 'react-native' ;
1414import { launchImageLibrary } from 'react-native-image-picker' ;
15+ import {
16+ AudioManager ,
17+ AudioRecorder ,
18+ AudioContext ,
19+ } from 'react-native-audio-api' ;
1520import { useIsFocused } from '@react-navigation/native' ;
1621import { useSafeAreaInsets } from 'react-native-safe-area-context' ;
1722import { models , useLLM } from 'react-native-executorch' ;
@@ -23,12 +28,14 @@ import Spinner from '../../components/Spinner';
2328import { GeneratingContext } from '../../context' ;
2429import SuggestedPrompts from '../../components/SuggestedPrompts' ;
2530import ErrorBanner from '../../components/ErrorBanner' ;
31+ import AudioWaveform from '../../components/AudioWaveform' ;
2632
2733const SUGGESTED_PROMPTS = [
2834 "What's in this image?" ,
2935 'Describe this scene in detail' ,
3036 'What objects can you see?' ,
3137 'What text appears in this image?' ,
38+ 'Transcribe the audio' ,
3239] ;
3340import { useLLMStats } from '../../hooks/useLLMStats' ;
3441import { StatsBar } from '../../components/StatsBar' ;
@@ -46,12 +53,18 @@ function MultimodalLLMScreen() {
4653 const textInputRef = useRef < TextInput > ( null ) ;
4754 const { setGlobalGenerating } = useContext ( GeneratingContext ) ;
4855
49- // Added error state
50- const [ error , setError ] = useState < string | null > ( null ) ;
56+ const [ audioBuffer , setAudioBuffer ] = useState < Float32Array | null > ( null ) ;
57+ const [ audioLabel , setAudioLabel ] = useState < string | null > ( null ) ;
58+ const [ audioUrl , setAudioUrl ] = useState ( '' ) ;
59+ const [ isFetchingAudio , setIsFetchingAudio ] = useState ( false ) ;
60+ const [ isRecording , setIsRecording ] = useState ( false ) ;
61+ const [ hasMicPermission , setHasMicPermission ] = useState ( false ) ;
62+ const recorder = useRef ( new AudioRecorder ( ) ) ;
63+ const recordChunks = useRef < Float32Array [ ] > ( [ ] ) ;
5164
52- const vlm = useLLM ( {
53- model : models . llm . lfm2_5_vl_1_6b ( ) ,
54- } ) ;
65+ const [ error , setError ] = useState < string | null > ( null ) ;
66+ const model = models . llm . gemma4_e2b_multimodal ( ) ;
67+ const vlm = useLLM ( { model : model } ) ;
5568 const tokenCount = vlm . isReady ? vlm . getGeneratedTokenCount ( ) : 0 ;
5669 const { stats, onMessageSend } = useLLMStats (
5770 vlm . response ,
@@ -68,6 +81,95 @@ function MultimodalLLMScreen() {
6881 if ( vlm . error ) setError ( String ( vlm . error ) ) ;
6982 } , [ vlm . error ] ) ;
7083
84+ useEffect ( ( ) => {
85+ AudioManager . setAudioSessionOptions ( {
86+ iosCategory : 'playAndRecord' ,
87+ iosMode : 'spokenAudio' ,
88+ iosOptions : [ 'allowBluetoothHFP' , 'defaultToSpeaker' ] ,
89+ } ) ;
90+ ( async ( ) => {
91+ const status = await AudioManager . requestRecordingPermissions ( ) ;
92+ setHasMicPermission ( status === 'Granted' ) ;
93+ } ) ( ) ;
94+
95+ return ( ) => {
96+ if ( vlm . isGenerating ) vlm . interrupt ( ) ;
97+ // eslint-disable-next-line react-hooks/exhaustive-deps
98+ recorder . current . stop ( ) ;
99+ AudioManager . setAudioSessionActivity ( false ) ;
100+ } ;
101+ // eslint-disable-next-line react-hooks/exhaustive-deps
102+ } , [ ] ) ;
103+
104+ const loadAudioFromUrl = async ( ) => {
105+ const url = audioUrl . trim ( ) ;
106+ if ( ! url ) return ;
107+ setIsFetchingAudio ( true ) ;
108+ try {
109+ const ctx = new AudioContext ( { sampleRate : 16000 } ) ;
110+ const decoded = await ctx . decodeAudioData ( url ) ;
111+ const pcm = decoded . getChannelData ( 0 ) ;
112+ const name = url . split ( '/' ) . pop ( ) || 'audio' ;
113+ setAudioBuffer ( pcm ) ;
114+ setAudioLabel ( `${ name } · ${ ( pcm . length / 16000 ) . toFixed ( 1 ) } s` ) ;
115+ } catch ( e ) {
116+ setError ( e instanceof Error ? e . message : String ( e ) ) ;
117+ } finally {
118+ setIsFetchingAudio ( false ) ;
119+ }
120+ } ;
121+
122+ const startRecording = async ( ) => {
123+ if ( ! hasMicPermission ) {
124+ setError ( 'Microphone permission denied. Please enable it in Settings.' ) ;
125+ return ;
126+ }
127+ recordChunks . current = [ ] ;
128+ const sampleRate = 16000 ;
129+ recorder . current . onAudioReady (
130+ { sampleRate, bufferLength : 0.1 * sampleRate , channelCount : 1 } ,
131+ ( { buffer } ) => {
132+ recordChunks . current . push ( new Float32Array ( buffer . getChannelData ( 0 ) ) ) ;
133+ }
134+ ) ;
135+ try {
136+ const ok = await AudioManager . setAudioSessionActivity ( true ) ;
137+ if ( ! ok ) {
138+ setError ( 'Cannot start audio session' ) ;
139+ return ;
140+ }
141+ const result = recorder . current . start ( ) ;
142+ if ( result . status === 'error' ) {
143+ setError ( `Recording problems: ${ result . message } ` ) ;
144+ return ;
145+ }
146+ setIsRecording ( true ) ;
147+ } catch ( e ) {
148+ setError ( e instanceof Error ? e . message : String ( e ) ) ;
149+ }
150+ } ;
151+
152+ const stopRecording = ( ) => {
153+ recorder . current . stop ( ) ;
154+ setIsRecording ( false ) ;
155+ const total = recordChunks . current . reduce ( ( n , c ) => n + c . length , 0 ) ;
156+ if ( total === 0 ) return ;
157+ const pcm = new Float32Array ( total ) ;
158+ let off = 0 ;
159+ for ( const c of recordChunks . current ) {
160+ pcm . set ( c , off ) ;
161+ off += c . length ;
162+ }
163+ recordChunks . current = [ ] ;
164+ setAudioBuffer ( pcm ) ;
165+ setAudioLabel ( `Recording · ${ ( pcm . length / 16000 ) . toFixed ( 1 ) } s` ) ;
166+ } ;
167+
168+ const clearAudio = ( ) => {
169+ setAudioBuffer ( null ) ;
170+ setAudioLabel ( null ) ;
171+ } ;
172+
71173 const pickImage = async ( ) => {
72174 try {
73175 const result = await launchImageLibrary ( { mediaType : 'photo' } ) ;
@@ -81,19 +183,27 @@ function MultimodalLLMScreen() {
81183 } ;
82184
83185 const sendMessage = async ( ) => {
84- if ( ! userInput . trim ( ) || vlm . isGenerating ) return ;
186+ if ( ! ( imageUri || audioBuffer || userInput . trim ( ) ) || vlm . isGenerating )
187+ return ;
85188 onMessageSend ( ) ;
86189 const text = userInput . trim ( ) ;
87190 setUserInput ( '' ) ;
88191 textInputRef . current ?. clear ( ) ;
89192 Keyboard . dismiss ( ) ;
90193 const currentImageUri = imageUri ;
194+ const currentAudio = audioBuffer ;
91195 setImageUri ( null ) ;
196+ setAudioBuffer ( null ) ;
197+ setAudioLabel ( null ) ;
92198 try {
93- await vlm . sendMessage (
94- text ,
95- currentImageUri ? { imagePath : currentImageUri } : undefined
96- ) ;
199+ const media =
200+ currentImageUri || currentAudio
201+ ? {
202+ ...( currentImageUri ? { imagePath : currentImageUri } : { } ) ,
203+ ...( currentAudio ? { audioBuffer : currentAudio } : { } ) ,
204+ }
205+ : undefined ;
206+ await vlm . sendMessage ( text , media ) ;
97207 } catch ( e ) {
98208 // Updated to set UI error instead of just console.error
99209 setError ( e instanceof Error ? e . message : String ( e ) ) ;
@@ -135,7 +245,9 @@ function MultimodalLLMScreen() {
135245 < View style = { styles . helloMessageContainer } >
136246 < Text style = { styles . helloText } > Hello! 👋</ Text >
137247 < Text style = { styles . bottomHelloText } >
138- Pick an image and ask me anything about it.
248+ { model . capabilities . find ( ( c ) => c === 'audio' )
249+ ? 'Say hi, or pick an image, and ask me anything about it.'
250+ : 'Pick an image and ask me anything about it.' }
139251 </ Text >
140252 < SuggestedPrompts
141253 prompts = { SUGGESTED_PROMPTS }
@@ -159,6 +271,48 @@ function MultimodalLLMScreen() {
159271 </ TouchableOpacity >
160272 ) }
161273
274+ { /* Audio URL input */ }
275+ < View style = { styles . audioUrlRow } >
276+ < TextInput
277+ placeholder = "Audio URL (mp3/wav/…)"
278+ placeholderTextColor = "#C1C6E5"
279+ style = { styles . audioUrlInput }
280+ value = { audioUrl }
281+ onChangeText = { setAudioUrl }
282+ autoCapitalize = "none"
283+ autoCorrect = { false }
284+ />
285+ < TouchableOpacity
286+ style = { [
287+ styles . audioUrlButton ,
288+ ( ! audioUrl . trim ( ) || isFetchingAudio || vlm . isGenerating ) &&
289+ styles . disabled ,
290+ ] }
291+ onPress = { loadAudioFromUrl }
292+ disabled = { ! audioUrl . trim ( ) || isFetchingAudio || vlm . isGenerating }
293+ >
294+ < Text style = { styles . audioUrlButtonText } >
295+ { isFetchingAudio ? '…' : 'Load' }
296+ </ Text >
297+ </ TouchableOpacity >
298+ </ View >
299+
300+ { /* Audio attachment strip */ }
301+ { audioLabel && (
302+ < View style = { styles . audioAttachmentContainer } >
303+ < View style = { styles . audioAttachmentRow } >
304+ < Text style = { styles . audioAttachmentText } > 🎵 { audioLabel } </ Text >
305+ < TouchableOpacity onPress = { clearAudio } >
306+ < Text style = { styles . audioAttachmentClear } > ✕</ Text >
307+ </ TouchableOpacity >
308+ </ View >
309+ < AudioWaveform
310+ buffer = { audioBuffer }
311+ style = { styles . audioWaveform }
312+ />
313+ </ View >
314+ ) }
315+
162316 < StatsBar stats = { stats } />
163317 < View
164318 style = { [
@@ -178,6 +332,17 @@ function MultimodalLLMScreen() {
178332 < Text style = { styles . imageButtonText } > 📷</ Text >
179333 </ TouchableOpacity >
180334
335+ { /* Mic record / stop button */ }
336+ < TouchableOpacity
337+ style = { styles . imageButton }
338+ onPress = { isRecording ? stopRecording : startRecording }
339+ disabled = { vlm . isGenerating }
340+ >
341+ < Text style = { styles . imageButtonText } >
342+ { isRecording ? '⏹️' : '🎤' }
343+ </ Text >
344+ </ TouchableOpacity >
345+
181346 < TextInput
182347 autoCorrect = { false }
183348 ref = { textInputRef }
@@ -198,14 +363,15 @@ function MultimodalLLMScreen() {
198363 onChangeText = { setUserInput }
199364 />
200365
201- { userInput . trim ( ) && ! vlm . isGenerating && (
202- < TouchableOpacity
203- style = { styles . sendChatTouchable }
204- onPress = { sendMessage }
205- >
206- < SendIcon height = { 24 } width = { 24 } padding = { 4 } margin = { 8 } />
207- </ TouchableOpacity >
208- ) }
366+ { ( imageUri || audioBuffer || userInput . trim ( ) ) &&
367+ ! vlm . isGenerating && (
368+ < TouchableOpacity
369+ style = { styles . sendChatTouchable }
370+ onPress = { sendMessage }
371+ >
372+ < SendIcon height = { 24 } width = { 24 } padding = { 4 } margin = { 8 } />
373+ </ TouchableOpacity >
374+ ) }
209375 { vlm . isGenerating && (
210376 < TouchableOpacity
211377 style = { styles . sendChatTouchable }
@@ -319,6 +485,71 @@ const styles = StyleSheet.create({
319485 fontFamily : 'regular' ,
320486 color : ColorPalette . blueDark ,
321487 } ,
488+ audioAttachmentContainer : {
489+ flexDirection : 'column' ,
490+ paddingHorizontal : 16 ,
491+ paddingVertical : 8 ,
492+ marginHorizontal : 16 ,
493+ marginBottom : 4 ,
494+ borderRadius : 8 ,
495+ borderWidth : 1 ,
496+ borderColor : ColorPalette . blueLight ,
497+ backgroundColor : '#fafbff' ,
498+ } ,
499+ audioAttachmentRow : {
500+ flexDirection : 'row' ,
501+ alignItems : 'center' ,
502+ justifyContent : 'space-between' ,
503+ } ,
504+ audioAttachmentText : {
505+ fontSize : 13 ,
506+ fontFamily : 'regular' ,
507+ color : ColorPalette . blueDark ,
508+ } ,
509+ audioAttachmentClear : {
510+ fontSize : 16 ,
511+ color : ColorPalette . blueDark ,
512+ paddingHorizontal : 8 ,
513+ } ,
514+ audioWaveform : {
515+ marginTop : 6 ,
516+ minWidth : 0 ,
517+ } ,
518+ audioUrlRow : {
519+ flexDirection : 'row' ,
520+ alignItems : 'center' ,
521+ marginHorizontal : 16 ,
522+ marginBottom : 4 ,
523+ } ,
524+ audioUrlInput : {
525+ flex : 1 ,
526+ padding : 10 ,
527+ borderTopLeftRadius : 8 ,
528+ borderBottomLeftRadius : 8 ,
529+ borderWidth : 1 ,
530+ borderColor : ColorPalette . blueLight ,
531+ borderRightWidth : 0 ,
532+ fontFamily : 'regular' ,
533+ fontSize : 13 ,
534+ color : ColorPalette . primary ,
535+ } ,
536+ audioUrlButton : {
537+ paddingVertical : 10 ,
538+ paddingHorizontal : 16 ,
539+ backgroundColor : ColorPalette . strongPrimary ,
540+ borderTopRightRadius : 8 ,
541+ borderBottomRightRadius : 8 ,
542+ justifyContent : 'center' ,
543+ alignItems : 'center' ,
544+ } ,
545+ audioUrlButtonText : {
546+ color : '#fff' ,
547+ fontFamily : 'medium' ,
548+ fontSize : 13 ,
549+ } ,
550+ disabled : {
551+ opacity : 0.5 ,
552+ } ,
322553 bottomContainer : {
323554 height : 100 ,
324555 width : '100%' ,
0 commit comments