feat(speech): add configurable persistent local Whisper (#29)

Author: configurowebmax

.gitignore

@@ -4,5 +4,6 @@ node_modules/
 .whisper-models/
 eng.traineddata
 dist/
+bin/
 .DS_Store
 *.log

README.md

@@ -34,8 +34,8 @@ It is free and open source. Processing stays on your machine, and the only thing
 
 - **Invisible overlay.** Windows stay out of Zoom, Google Meet, Microsoft Teams, Discord, and OBS captures. You see the answer, the call does not.
 - **Hidden during screen share.** When a share starts, the app can hide every window on its own.
-- **Real-time voice.** Speech is split on natural pauses instead of a fixed timer, so one spoken question stays one question. Filler phrases that Whisper invents on silence are dropped before they reach the model.
-- **Streamed answers.** Replies appear word by word as the model generates them, in both the chat and the floating window.
+- **Flexible local voice.** Choose manual start/stop capture or automatic voice-activity detection without fixed-timer sentence cuts.
+- **Configurable streamed answers.** Route voice replies to chat, the floating overlay, or both.
 - **Direct image analysis.** Screenshots go straight to Gemini for visual reasoning, with no slow OCR step in between.
 - **Session memory.** The whole conversation is remembered, so follow-ups, edge cases, and optimizations keep their context.
 - **Language aware.** Tailored answers for C++, C, Python, Java, and JavaScript.
@@ -116,8 +116,14 @@ AZURE_SPEECH_REGION=your_region
 # Local Whisper option
 WHISPER_COMMAND=whisper
 WHISPER_MODEL_DIR=.whisper-models
-WHISPER_MODEL=turbo
-WHISPER_LANGUAGE=en
+WHISPER_MODEL=small
+WHISPER_LANGUAGE=auto
+WHISPER_DEVICE=auto
+WHISPER_PYTHON=
+WHISPER_CAPTURE_MODE=vad
+WHISPER_RESPONSE_TARGET=both
+WHISPER_MANUAL_MAX_MS=90000
+WHISPER_GPU_IDLE_MS=60000
 ```
 
 Speech is optional. If no provider is configured, the microphone button hides itself across the app.
@@ -126,15 +132,15 @@ Speech is optional. If no provider is configured, the microphone button hides it
 
 You can use local Whisper for offline transcription or Azure Speech for a cloud option.
 
-For local Whisper, `./setup.sh` handles the full setup. It creates `.venv-whisper`, installs `openai-whisper`, points `.env` at the virtual environment, creates `.whisper-models`, and runs a quick speech test. You only need Python 3.10 or newer and ffmpeg on your system. Install those with `./setup.sh --install-system-deps`, or add `ffmpeg` and `sox` yourself.
+For local Whisper, `./setup.sh` handles the full setup. It creates `.venv-whisper`, installs `openai-whisper`, points `.env` at the virtual environment, creates `.whisper-models`, and runs a quick speech test. The app reads its own PCM WAV recordings directly; ffmpeg is only needed when transcribing other audio formats through the CLI fallback.
 
 For Azure Speech, create a Speech resource in the [Azure Portal](https://portal.azure.com/), then add the key and region to `.env` with `SPEECH_PROVIDER=azure`.
 
 ## How it works
 
-1. **Ask.** Speak the question or press the screenshot shortcut. The microphone listens for natural pauses on its own and does not cut you off mid sentence.
+1. **Ask.** Use automatic pause detection, choose manual start/stop capture in Settings, or use the screenshot shortcut.
 2. **Reason.** Gemini reads the audio or image with full conversation context and works toward a precise answer.
-3. **Answer.** The response streams into the overlay in real time, with formatted text and highlighted code.
+3. **Answer.** Voice responses stream to chat, the overlay, or both, according to Settings.
 
 ## Keyboard shortcuts
 
@@ -156,9 +162,9 @@ OpenCluely is under active development. The core is stable and improvements ship
 - Stealth overlay with a draggable command bar and a click through toggle
 - Hidden during screen share, with automatic hiding when a share begins
 - Screenshot capture with direct Gemini analysis, no OCR step
-- Real-time voice input that segments on natural pauses, not a blind timer
-- Utterance coalescing so one spoken question becomes one answer
-- Streamed answers that render word by word in the chat and the overlay
+- Configurable manual or VAD-driven voice capture
+- Persistent local Whisper worker with optional CUDA acceleration and idle GPU release
+- Configurable chat/overlay routing for streamed voice answers
 - Whisper hallucination filter that drops phantom phrases on silence
 - AI response window with markdown and syntax highlighting
 - Global shortcuts for capture, visibility, interaction, chat, and settings

env.example

@@ -20,6 +20,12 @@ WHISPER_COMMAND=whisper
 # app-data folder (recommended). Set an ABSOLUTE path to override; a relative
 # path is ignored because it cannot be resolved reliably in packaged builds.
 # WHISPER_MODEL_DIR=
-WHISPER_MODEL=turbo
-WHISPER_LANGUAGE=en
+WHISPER_MODEL=small
+WHISPER_LANGUAGE=auto
 WHISPER_SEGMENT_MS=4000
+WHISPER_DEVICE=auto
+WHISPER_PYTHON=
+WHISPER_CAPTURE_MODE=vad
+WHISPER_RESPONSE_TARGET=both
+WHISPER_MANUAL_MAX_MS=90000
+WHISPER_GPU_IDLE_MS=60000

main.js

@@ -1,5 +1,6 @@
 const path = require("path");
 const fs = require("fs");
+const { fileURLToPath } = require("url");
 const { app, BrowserWindow, globalShortcut, session, ipcMain } = require("electron");
 
 // ── Resolve a stable .env location ──
@@ -332,12 +333,56 @@ class ApplicationController {
   }
 
   setupPermissions() {
-    session.defaultSession.setPermissionRequestHandler(
-      (webContents, permission, callback) => {
-        const allowedPermissions = ["microphone", "camera", "display-capture"];
-        const granted = allowedPermissions.includes(permission);
+    const appSession = session.defaultSession;
+    const isTrustedAppContents = (webContents) => {
+      if (!webContents || webContents.isDestroyed()) {
+        return false;
+      }
+      try {
+        const pagePath = path.resolve(fileURLToPath(webContents.getURL()));
+        const appRoot = path.resolve(__dirname);
+        const normalizeForComparison = (value) => process.platform === "win32"
+          ? value.toLowerCase()
+          : value;
+        const page = normalizeForComparison(pagePath);
+        const root = normalizeForComparison(appRoot + path.sep);
+        return page.startsWith(root);
+      } catch (_) {
+        return false;
+      }
+    };
 
-        logger.debug("Permission request", { permission, granted });
+    // Electron exposes camera/microphone access as the single `media`
+    // permission. The requested device type is provided separately in details.
+    appSession.setPermissionCheckHandler(
+      (webContents, permission, _requestingOrigin, details = {}) => {
+        if (!isTrustedAppContents(webContents)) {
+          return false;
+        }
+        if (permission === "media") {
+          return !details.mediaType || details.mediaType === "audio";
+        }
+        return permission === "display-capture";
+      }
+    );
+
+    appSession.setPermissionRequestHandler(
+      (webContents, permission, callback, details = {}) => {
+        let granted = false;
+        if (isTrustedAppContents(webContents)) {
+          if (permission === "media") {
+            const mediaTypes = Array.isArray(details.mediaTypes) ? details.mediaTypes : [];
+            granted = mediaTypes.length === 0 || mediaTypes.includes("audio");
+          } else {
+            granted = permission === "display-capture";
+          }
+        }
+
+        logger.debug("Permission request", {
+          permission,
+          mediaTypes: details.mediaTypes || [],
+          granted
+        });
         callback(granted);
       }
     );
@@ -373,15 +418,11 @@ class ApplicationController {
 
   setupServiceEventHandlers() {
     speechService.on("recording-started", () => {
-      BrowserWindow.getAllWindows().forEach((window) => {
-        window.webContents.send("recording-started");
-      });
+      windowManager.handleRecordingStarted();
     });
 
     speechService.on("recording-stopped", () => {
-      BrowserWindow.getAllWindows().forEach((window) => {
-        window.webContents.send("recording-stopped");
-      });
+      windowManager.handleRecordingStopped();
     });
 
     speechService.on("transcription", (text) => {
@@ -787,7 +828,7 @@ class ApplicationController {
       try {
         const installer = this.getWhisperInstaller();
         const sender = event.sender;
-        const result = await installer.downloadModel(modelName || 'turbo', {
+        const result = await installer.downloadModel(modelName || 'small', {
           onProgress: (line) => {
             try { sender.send("install-progress", line); } catch (_) { /* ignore */ }
           },
@@ -917,7 +958,6 @@ class ApplicationController {
     if (currentStatus.isRecording) {
       try {
         speechService.stopRecording();
-        windowManager.hideChatWindow();
         logger.info("Speech recognition stopped via global shortcut");
       } catch (error) {
         logger.error("Error stopping speech recognition:", error);
@@ -1193,19 +1233,26 @@ class ApplicationController {
       return;
     }
 
-    // Show the live transcript right away in all windows.
+    // Route speech UI events according to the user's response-target setting.
     sessionManager.addUserInput(fragment, 'speech');
-    BrowserWindow.getAllWindows().forEach((window) => {
-      window.webContents.send("transcription-received", { text: fragment });
-    });
+    this.sendToVoiceResponseWindows("transcription-received", { text: fragment });
 
     this._utteranceBuffer = this._utteranceBuffer
       ? `${this._utteranceBuffer} ${fragment}`
       : fragment;
 
     if (this._utteranceTimer) {
       clearTimeout(this._utteranceTimer);
+      this._utteranceTimer = null;
     }
+
+    // Manual capture emits one complete transcript after the user presses stop,
+    // so no debounce/coalescing delay is needed.
+    if (speechService.isManualCaptureMode()) {
+      this.dispatchCoalescedUtterance();
+      return;
+    }
+
     this._utteranceTimer = setTimeout(() => {
       this._utteranceTimer = null;
       this.dispatchCoalescedUtterance();
@@ -1278,26 +1325,25 @@ class ApplicationController {
       const skillsRequiringProgrammingLanguage = ['dsa'];
       const needsProgrammingLanguage = skillsRequiringProgrammingLanguage.includes(this.activeSkill);
 
-      // Stream the answer so it renders progressively in the chat + overlay.
+      // Stream the answer progressively to the configured speech target.
       // A unique messageId ties the start/chunk/final events to one bubble so
       // the UI never duplicates or interleaves concurrent responses.
       this._responseSeq = (this._responseSeq || 0) + 1;
       messageId = `tr-${Date.now()}-${this._responseSeq}`;
-      windowManager.broadcastToAllWindows("transcription-llm-response-start", {
+      this.sendToVoiceResponseWindows("transcription-llm-response-start", {
         messageId,
         skill: this.activeSkill
       });
-      // Surface the overlay immediately so streamed tokens are visible there
-      // too, instead of the overlay only appearing once the full answer lands.
-      windowManager.showLLMLoading();
-
+      if (this.shouldShowVoiceOverlay()) {
+        windowManager.showLLMLoading();
+      }
       const llmResult = await llmService.processTranscriptionWithIntelligentResponseStream(
         cleanText,
         this.activeSkill,
         sessionHistory.recent,
         needsProgrammingLanguage ? this.codingLanguage : null,
         (delta) => {
-          windowManager.broadcastToAllWindows("transcription-llm-response-chunk", {
+          this.sendToVoiceResponseWindows("transcription-llm-response-chunk", {
             messageId,
             delta
           });
@@ -1313,18 +1359,15 @@ class ApplicationController {
         isTranscriptionResponse: true
       });
 
-      // Send response to chat windows
-      this.broadcastTranscriptionLLMResponse(llmResult);
-
-      // Also display in the overlay (LLM response) window so the answer
-      // appears in both the chat panel and the floating overlay, mirroring
-      // the behaviour of screenshot/image responses.
-      windowManager.showLLMResponse(llmResult.response, {
-        skill: this.activeSkill,
-        processingTime: llmResult.metadata.processingTime,
-        usedFallback: llmResult.metadata.usedFallback,
-        isTranscriptionResponse: true
-      });
+      this.sendTranscriptionLLMResponseToVoiceTargets(llmResult);
+      if (this.shouldShowVoiceOverlay()) {
+        windowManager.showLLMResponse(llmResult.response, {
+          skill: this.activeSkill,
+          processingTime: llmResult.metadata.processingTime,
+          usedFallback: llmResult.metadata.usedFallback,
+          isTranscriptionResponse: true
+        });
+      }
 
       logger.info("Transcription LLM response completed", {
         responseLength: llmResult.response.length,
@@ -1344,7 +1387,7 @@ class ApplicationController {
       // Try to provide a fallback response
       try {
         const fallbackResult = llmService.generateIntelligentFallbackResponse(text, this.activeSkill);
-        // Carry the streaming messageId so the chat/overlay replace the live
+        // Carry the streaming messageId so the target replaces the live
         // bubble instead of leaving it stuck and appending a duplicate.
         if (messageId) {
           fallbackResult.metadata = { ...fallbackResult.metadata, messageId };
@@ -1358,14 +1401,15 @@ class ApplicationController {
           fallbackReason: error.message
         });
 
-        this.broadcastTranscriptionLLMResponse(fallbackResult);
-        // Mirror to overlay window for consistency
-        windowManager.showLLMResponse(fallbackResult.response, {
-          skill: this.activeSkill,
-          processingTime: fallbackResult.metadata.processingTime,
-          usedFallback: true,
-          isTranscriptionResponse: true
-        });
+        this.sendTranscriptionLLMResponseToVoiceTargets(fallbackResult);
+        if (this.shouldShowVoiceOverlay()) {
+          windowManager.showLLMResponse(fallbackResult.response, {
+            skill: this.activeSkill,
+            processingTime: fallbackResult.metadata.processingTime,
+            usedFallback: true,
+            isTranscriptionResponse: true
+          });
+        }
         logger.info("Used fallback response for transcription", {
           skill: this.activeSkill,
           fallbackResponse: fallbackResult.response
@@ -1445,6 +1489,48 @@ class ApplicationController {
     windowManager.broadcastToAllWindows("transcription-llm-response", broadcastData);
   }
 
+  sendToChatWindow(channel, data) {
+    const chatWindow = windowManager.getWindow("chat");
+    if (!chatWindow || chatWindow.isDestroyed()) {
+      logger.warn("Chat window unavailable for speech event", { channel });
+      return;
+    }
+    chatWindow.webContents.send(channel, data);
+  }
+
+  getVoiceResponseTarget() {
+    const configured = String(process.env.WHISPER_RESPONSE_TARGET || 'both').trim().toLowerCase();
+    return ['chat', 'overlay', 'both'].includes(configured) ? configured : 'both';
+  }
+
+  shouldShowVoiceOverlay() {
+    return ['overlay', 'both'].includes(this.getVoiceResponseTarget());
+  }
+
+  sendToVoiceResponseWindows(channel, data) {
+    const target = this.getVoiceResponseTarget();
+    if (target === 'chat' || target === 'both') {
+      this.sendToChatWindow(channel, data);
+    }
+    if (target === 'overlay' || target === 'both') {
+      const responseWindow = windowManager.getWindow("llmResponse");
+      if (responseWindow && !responseWindow.isDestroyed()) {
+        responseWindow.webContents.send(channel, data);
+      }
+    }
+  }
+
+  sendTranscriptionLLMResponseToVoiceTargets(llmResult) {
+    const data = {
+      response: llmResult.response,
+      metadata: llmResult.metadata,
+      messageId: llmResult.metadata && llmResult.metadata.messageId,
+      skill: this.activeSkill,
+      isTranscriptionResponse: true
+    };
+    this.sendToVoiceResponseWindows("transcription-llm-response", data);
+  }
+
   onWindowAllClosed() {
     if (process.platform !== "darwin") {
       app.quit();
@@ -1474,6 +1560,7 @@ class ApplicationController {
 
   onWillQuit() {
     globalShortcut.unregisterAll();
+    speechService.shutdown();
     windowManager.destroyAllWindows();
 
     const sessionStats = sessionManager.getMemoryUsage();
@@ -1512,8 +1599,12 @@ class ApplicationController {
       azureKey: process.env.AZURE_SPEECH_KEY || "",
       azureRegion: process.env.AZURE_SPEECH_REGION || "",
       whisperCommand: process.env.WHISPER_COMMAND || "",
-      whisperModel: process.env.WHISPER_MODEL || "turbo",
-      whisperLanguage: process.env.WHISPER_LANGUAGE || "en",
+      whisperModel: process.env.WHISPER_MODEL || "small",
+      whisperLanguage: process.env.WHISPER_LANGUAGE || "auto",
+      whisperDevice: process.env.WHISPER_DEVICE || "auto",
+      whisperCaptureMode: process.env.WHISPER_CAPTURE_MODE ||
+        (process.env.WHISPER_MANUAL_CAPTURE === "true" ? "manual" : "vad"),
+      whisperResponseTarget: process.env.WHISPER_RESPONSE_TARGET || "both",
       whisperSegmentMs: process.env.WHISPER_SEGMENT_MS || "4000",
       geminiKey: process.env.GEMINI_API_KEY || "",
 
@@ -1572,6 +1663,15 @@ class ApplicationController {
       if (settings.whisperLanguage !== undefined) {
         envUpdates.WHISPER_LANGUAGE = settings.whisperLanguage;
       }
+      if (["auto", "cpu", "cuda"].includes(settings.whisperDevice)) {
+        envUpdates.WHISPER_DEVICE = settings.whisperDevice;
+      }
+      if (["manual", "vad"].includes(settings.whisperCaptureMode)) {
+        envUpdates.WHISPER_CAPTURE_MODE = settings.whisperCaptureMode;
+      }
+      if (["chat", "overlay", "both"].includes(settings.whisperResponseTarget)) {
+        envUpdates.WHISPER_RESPONSE_TARGET = settings.whisperResponseTarget;
+      }
       if (settings.whisperSegmentMs !== undefined) {
         envUpdates.WHISPER_SEGMENT_MS = String(settings.whisperSegmentMs);
       }