fix to unresponsive code bug

Author: mkopcins

apps/llm/app/multimodal_llm/index.tsx

@@ -28,6 +28,7 @@ import Spinner from '../../components/Spinner';
 import { GeneratingContext } from '../../context';
 import SuggestedPrompts from '../../components/SuggestedPrompts';
 import ErrorBanner from '../../components/ErrorBanner';
+import AudioWaveform from '../../components/AudioWaveform';
 
 const SUGGESTED_PROMPTS = [
   "What's in this image?",
@@ -291,10 +292,16 @@ function MultimodalLLMScreen() {
           {/* Audio attachment strip */}
           {audioLabel && (
             <View style={styles.audioAttachmentContainer}>
-              <Text style={styles.audioAttachmentText}>🎵 {audioLabel}</Text>
-              <TouchableOpacity onPress={clearAudio}>
-                <Text style={styles.audioAttachmentClear}>✕</Text>
-              </TouchableOpacity>
+              <View style={styles.audioAttachmentRow}>
+                <Text style={styles.audioAttachmentText}>🎵 {audioLabel}</Text>
+                <TouchableOpacity onPress={clearAudio}>
+                  <Text style={styles.audioAttachmentClear}>✕</Text>
+                </TouchableOpacity>
+              </View>
+              <AudioWaveform
+                buffer={audioBuffer}
+                style={styles.audioWaveform}
+              />
             </View>
           )}
 
@@ -471,9 +478,7 @@ const styles = StyleSheet.create({
     color: ColorPalette.blueDark,
   },
   audioAttachmentContainer: {
-    flexDirection: 'row',
-    alignItems: 'center',
-    justifyContent: 'space-between',
+    flexDirection: 'column',
     paddingHorizontal: 16,
     paddingVertical: 8,
     marginHorizontal: 16,
@@ -483,6 +488,11 @@ const styles = StyleSheet.create({
     borderColor: ColorPalette.blueLight,
     backgroundColor: '#fafbff',
   },
+  audioAttachmentRow: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    justifyContent: 'space-between',
+  },
   audioAttachmentText: {
     fontSize: 13,
     fontFamily: 'regular',
@@ -493,6 +503,10 @@ const styles = StyleSheet.create({
     color: ColorPalette.blueDark,
     paddingHorizontal: 8,
   },
+  audioWaveform: {
+    marginTop: 6,
+    minWidth: 0,
+  },
   audioUrlRow: {
     flexDirection: 'row',
     alignItems: 'center',

apps/llm/components/MessageItem.tsx

@@ -11,6 +11,7 @@ import MarkdownComponent from './MarkdownComponent';
 import LlamaIcon from '../assets/icons/llama_icon.svg';
 import ColorPalette from '../colors';
 import { Message } from 'react-native-executorch';
+import AudioWaveform from './AudioWaveform';
 
 interface MessageItemProps {
   message: Message;
@@ -43,6 +44,12 @@ const MessageItem = memo(({ message, deleteMessage }: MessageItemProps) => {
             resizeMode="contain"
           />
         )}
+        {message.audioWaveform && (
+          <AudioWaveform
+            buffer={message.audioWaveform}
+            style={styles.userMessageWaveform}
+          />
+        )}
         <MarkdownComponent text={message.content} />
       </View>
     </View>
@@ -103,6 +110,9 @@ const styles = StyleSheet.create({
     borderRadius: 6,
     marginBottom: 6,
   },
+  userMessageWaveform: {
+    marginBottom: 6,
+  },
   aiMessageIconContainer: {
     backgroundColor: ColorPalette.seaBlueLight,
     height: 32,

packages/react-native-executorch/src/controllers/LLMController.ts

@@ -297,6 +297,7 @@ export class LLMController {
     const hasImages = !!imagePaths && imagePaths.length > 0;
     const hasAudio = !!audioWaveforms && audioWaveforms.length > 0;
     try {
+      console.log('kappa');
       this.isGeneratingCallback(true);
       this.nativeModule.reset();
       let response: string;
@@ -360,6 +361,8 @@ export class LLMController {
     messages: Message[],
     tools?: LLMTool[]
   ): Promise<string> {
+    let s = performance.now();
+    console.log('kappa', performance.now() - s);
     if (!this._isReady) {
       throw new RnExecutorchError(
         RnExecutorchErrorCode.ModuleNotLoaded,
@@ -384,13 +387,13 @@ export class LLMController {
     const audioWaveforms = messages
       .filter((m) => m.audioWaveform)
       .map((m) => m.audioWaveform!);
-
     const renderedChat: string = this.applyChatTemplate(
       messages,
       this.tokenizerConfig,
       tools,
       { tools_in_user_message: false, add_generation_prompt: true }
     );
+    console.log('kappa', performance.now() - s);
 
     return await this.forward(
       renderedChat,
@@ -553,6 +556,8 @@ function messagesForChatTemplate(messages: Message[]): any[] {
     if (hasImage) parts.push({ type: 'image' });
     if (hasAudio) parts.push({ type: 'audio' });
     parts.push({ type: 'text', text: m.content });
-    return { ...m, content: parts };
+    // Drop the Float32Array on the clone only — passing it into the Jinja
+    // template engine slows render past 3s. Don't mutate m;
+    return { ...m, content: parts, audioWaveform: undefined };
   });
 }