fix to unresponsive code bug

Author: mkopcins

apps/llm/app/multimodal_llm/index.tsx

@@ -28,6 +28,7 @@ import Spinner from '../../components/Spinner';
 import { GeneratingContext } from '../../context';
 import SuggestedPrompts from '../../components/SuggestedPrompts';
 import ErrorBanner from '../../components/ErrorBanner';
+import AudioWaveform from '../../components/AudioWaveform';
 
 const SUGGESTED_PROMPTS = [
   "What's in this image?",
@@ -291,10 +292,16 @@ function MultimodalLLMScreen() {
           {/* Audio attachment strip */}
           {audioLabel && (
             <View style={styles.audioAttachmentContainer}>
-              <Text style={styles.audioAttachmentText}>🎵 {audioLabel}</Text>
-              <TouchableOpacity onPress={clearAudio}>
-                <Text style={styles.audioAttachmentClear}>✕</Text>
-              </TouchableOpacity>
+              <View style={styles.audioAttachmentRow}>
+                <Text style={styles.audioAttachmentText}>🎵 {audioLabel}</Text>
+                <TouchableOpacity onPress={clearAudio}>
+                  <Text style={styles.audioAttachmentClear}>✕</Text>
+                </TouchableOpacity>
+              </View>
+              <AudioWaveform
+                buffer={audioBuffer}
+                style={styles.audioWaveform}
+              />
             </View>
           )}
 
@@ -471,9 +478,7 @@ const styles = StyleSheet.create({
     color: ColorPalette.blueDark,
   },
   audioAttachmentContainer: {
-    flexDirection: 'row',
-    alignItems: 'center',
-    justifyContent: 'space-between',
+    flexDirection: 'column',
     paddingHorizontal: 16,
     paddingVertical: 8,
     marginHorizontal: 16,
@@ -483,6 +488,11 @@ const styles = StyleSheet.create({
     borderColor: ColorPalette.blueLight,
     backgroundColor: '#fafbff',
   },
+  audioAttachmentRow: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    justifyContent: 'space-between',
+  },
   audioAttachmentText: {
     fontSize: 13,
     fontFamily: 'regular',
@@ -493,6 +503,10 @@ const styles = StyleSheet.create({
     color: ColorPalette.blueDark,
     paddingHorizontal: 8,
   },
+  audioWaveform: {
+    marginTop: 6,
+    minWidth: 0,
+  },
   audioUrlRow: {
     flexDirection: 'row',
     alignItems: 'center',

apps/llm/components/AudioWaveform.tsx

@@ -0,0 +1,57 @@
+import { useMemo } from 'react';
+import { StyleProp, StyleSheet, View, ViewStyle } from 'react-native';
+import ColorPalette from '../colors';
+
+interface AudioWaveformProps {
+  buffer: Float32Array | null | undefined;
+  style?: StyleProp<ViewStyle>;
+}
+
+const NUM_BARS = 32;
+
+export default function AudioWaveform({ buffer, style }: AudioWaveformProps) {
+  const bars = useMemo(() => {
+    if (!buffer || buffer.length === 0) return null;
+    const chunkSize = Math.max(1, Math.floor(buffer.length / NUM_BARS));
+    const peaks: number[] = [];
+    let max = 0;
+    for (let i = 0; i < NUM_BARS; i++) {
+      const start = i * chunkSize;
+      const end = Math.min(start + chunkSize, buffer.length);
+      let peak = 0;
+      for (let j = start; j < end; j++) {
+        const v = Math.abs(buffer[j] ?? 0);
+        if (v > peak) peak = v;
+      }
+      peaks.push(peak);
+      if (peak > max) max = peak;
+    }
+    return max > 0 ? peaks.map((p) => p / max) : peaks;
+  }, [buffer]);
+
+  if (!bars) return null;
+
+  return (
+    <View style={[styles.container, style]}>
+      {bars.map((amp, i) => (
+        <View key={i} style={[styles.bar, { height: 2 + amp * 14 }]} />
+      ))}
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    height: 16,
+    minWidth: 160,
+    gap: 2,
+  },
+  bar: {
+    flex: 1,
+    borderRadius: 1,
+    backgroundColor: ColorPalette.blueDark,
+    opacity: 0.35,
+  },
+});

apps/llm/components/MessageItem.tsx

@@ -11,6 +11,7 @@ import MarkdownComponent from './MarkdownComponent';
 import LlamaIcon from '../assets/icons/llama_icon.svg';
 import ColorPalette from '../colors';
 import { Message } from 'react-native-executorch';
+import AudioWaveform from './AudioWaveform';
 
 interface MessageItemProps {
   message: Message;
@@ -43,6 +44,12 @@ const MessageItem = memo(({ message, deleteMessage }: MessageItemProps) => {
             resizeMode="contain"
           />
         )}
+        {message.audioWaveform && (
+          <AudioWaveform
+            buffer={message.audioWaveform}
+            style={styles.userMessageWaveform}
+          />
+        )}
         <MarkdownComponent text={message.content} />
       </View>
     </View>
@@ -103,6 +110,9 @@ const styles = StyleSheet.create({
     borderRadius: 6,
     marginBottom: 6,
   },
+  userMessageWaveform: {
+    marginBottom: 6,
+  },
   aiMessageIconContainer: {
     backgroundColor: ColorPalette.seaBlueLight,
     height: 32,

packages/react-native-executorch/src/controllers/LLMController.ts

@@ -297,6 +297,7 @@ export class LLMController {
     const hasImages = !!imagePaths && imagePaths.length > 0;
     const hasAudio = !!audioWaveforms && audioWaveforms.length > 0;
     try {
+      console.log('kappa');
       this.isGeneratingCallback(true);
       this.nativeModule.reset();
       let response: string;
@@ -360,6 +361,8 @@ export class LLMController {
     messages: Message[],
     tools?: LLMTool[]
   ): Promise<string> {
+    let s = performance.now();
+    console.log('kappa', performance.now() - s);
     if (!this._isReady) {
       throw new RnExecutorchError(
         RnExecutorchErrorCode.ModuleNotLoaded,
@@ -384,13 +387,13 @@ export class LLMController {
     const audioWaveforms = messages
       .filter((m) => m.audioWaveform)
       .map((m) => m.audioWaveform!);
-
     const renderedChat: string = this.applyChatTemplate(
       messages,
       this.tokenizerConfig,
       tools,
       { tools_in_user_message: false, add_generation_prompt: true }
     );
+    console.log('kappa', performance.now() - s);
 
     return await this.forward(
       renderedChat,
@@ -553,6 +556,8 @@ function messagesForChatTemplate(messages: Message[]): any[] {
     if (hasImage) parts.push({ type: 'image' });
     if (hasAudio) parts.push({ type: 'audio' });
     parts.push({ type: 'text', text: m.content });
-    return { ...m, content: parts };
+    // Drop the Float32Array on the clone only — passing it into the Jinja
+    // template engine slows render past 3s. Don't mutate m;
+    return { ...m, content: parts, audioWaveform: undefined };
   });
 }