Merge pull request #199 from off-grid-ai/perf/desktop-optimizations-h3zn29

Documents and project knowledge bases on one engine; voice and timeline polish

Author: alichherawalla

electron-builder.yml

@@ -17,6 +17,8 @@ afterSign: scripts/notarize-mac.mjs
 asarUnpack:
   - 'node_modules/@img/sharp-libvips-linux-x64/lib/*.so*'
   - 'node_modules/**/onnxruntime-node/bin/**/*'
+  # ONNX Runtime opens model files by path; it cannot read them inside app.asar.
+  - 'node_modules/@gutenye/ocr-models/assets/**/*'
 directories:
   buildResources: build
 files:

electron.vite.config.ts

@@ -134,6 +134,8 @@ export default defineConfig({
         input: {
           index: resolve('src/main/index.ts'),
           'embeddings-worker': resolve('src/main/embeddings-worker.ts'),
+          'documents-extract-worker': resolve('src/main/documents/extract-worker.ts'),
+          'documents-ocr-worker': resolve('src/main/documents/ocr-worker.ts'),
           'tts-onnx-worker': resolve('src/main/tts-onnx-worker.ts')
         }
       }

electron/accessibility/ocr.swift

@@ -1,10 +1,16 @@
 import Foundation
 import Vision
 import AppKit
+import PDFKit
 
 // OCR a single image file with the macOS Vision framework and print the
 // recognized text, one line per detected block. Fast, accurate, on-device,
 // AGPL-safe (no external service). Usage: ocr [--json] <image-path>
+//
+// PDF mode: ocr --pdf <file.pdf> --pages 3,4,9
+// Renders each page with PDFKit onto opaque white (a transparent background reads as nothing)
+// at 200 DPI, long side capped at 2800 px, and prints one JSON line per page: {"page":3,"text":"…"}.
+// One process reads many pages, so a scanned book does not pay a process start per page.
 
 struct PixelBounds: Codable {
     let x: Int
@@ -25,6 +31,84 @@ struct OcrDocument: Codable {
     let blocks: [TextBlock]
 }
 
+func recognizeText(
+    _ cgImage: CGImage,
+    recognitionLevel: VNRequestTextRecognitionLevel,
+    languageCorrection: Bool
+) throws -> [VNRecognizedTextObservation] {
+    let request = VNRecognizeTextRequest()
+    request.recognitionLevel = recognitionLevel
+    request.usesLanguageCorrection = languageCorrection
+    let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
+    try handler.perform([request])
+    return request.results ?? []
+}
+
+
+struct PdfPageText: Codable {
+    let page: Int
+    let text: String
+}
+
+func recognizePageText(_ image: CGImage) -> String {
+    let observations =
+        (try? recognizeText(image, recognitionLevel: .accurate, languageCorrection: true))
+        ?? (try? recognizeText(image, recognitionLevel: .accurate, languageCorrection: false))
+        ?? []
+    return observations.compactMap { $0.topCandidates(1).first?.string }.joined(separator: "\n")
+}
+
+func renderPage(_ page: PDFPage) -> CGImage? {
+    let bounds = page.bounds(for: .mediaBox)
+    guard bounds.width > 0, bounds.height > 0 else { return nil }
+    let scale = min(200.0 / 72.0, 2800.0 / max(bounds.width, bounds.height))
+    let width = Int((bounds.width * scale).rounded())
+    let height = Int((bounds.height * scale).rounded())
+    guard let context = CGContext(
+        data: nil,
+        width: width,
+        height: height,
+        bitsPerComponent: 8,
+        bytesPerRow: 0,
+        space: CGColorSpaceCreateDeviceGray(),
+        bitmapInfo: CGImageAlphaInfo.none.rawValue
+    ) else { return nil }
+    context.setFillColor(gray: 1, alpha: 1)
+    context.fill(CGRect(x: 0, y: 0, width: width, height: height))
+    context.scaleBy(x: scale, y: scale)
+    context.translateBy(x: -bounds.origin.x, y: -bounds.origin.y)
+    page.draw(with: .mediaBox, to: context)
+    return context.makeImage()
+}
+
+if CommandLine.arguments.count > 2 && CommandLine.arguments[1] == "--pdf" {
+    let pdfPath = CommandLine.arguments[2]
+    var pages: [Int] = []
+    if let flag = CommandLine.arguments.firstIndex(of: "--pages"), flag + 1 < CommandLine.arguments.count {
+        pages = CommandLine.arguments[flag + 1].split(separator: ",").compactMap { Int($0) }
+    }
+    guard let document = PDFDocument(url: URL(fileURLWithPath: pdfPath)) else {
+        FileHandle.standardError.write("could not open PDF\n".data(using: .utf8)!)
+        exit(1)
+    }
+    if pages.isEmpty { pages = Array(1...max(1, document.pageCount)) }
+    let encoder = JSONEncoder()
+    for number in pages {
+        autoreleasepool {
+            var text = ""
+            if let page = document.page(at: number - 1), let image = renderPage(page) {
+                text = recognizePageText(image)
+            }
+            if let data = try? encoder.encode(PdfPageText(page: number, text: text)),
+               let line = String(data: data, encoding: .utf8) {
+                print(line)
+                fflush(stdout)
+            }
+        }
+    }
+    exit(0)
+}
+
 guard CommandLine.arguments.count > 1 else {
     FileHandle.standardError.write("usage: ocr <image-path>\n".data(using: .utf8)!)
     exit(1)
@@ -43,19 +127,6 @@ guard let image = NSImage(contentsOfFile: path),
     exit(1)
 }
 
-func recognizeText(
-    _ cgImage: CGImage,
-    recognitionLevel: VNRequestTextRecognitionLevel,
-    languageCorrection: Bool
-) throws -> [VNRecognizedTextObservation] {
-    let request = VNRecognizeTextRequest()
-    request.recognitionLevel = recognitionLevel
-    request.usesLanguageCorrection = languageCorrection
-    let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
-    try handler.perform([request])
-    return request.results ?? []
-}
-
 let results: [VNRecognizedTextObservation]
 do {
     do {

integration-tests/conversation-rename.dbtest.ts

@@ -87,6 +87,11 @@ function installApi(): void {
         finishedAt: null
       }),
       onRagStream: () => () => undefined,
+      documents: {
+        list: async () => [],
+        setExcluded: async () => undefined,
+        onProgress: () => () => undefined
+      },
       onImageGenProgress: () => () => undefined,
       onImageGenJobState: () => () => undefined,
       onImageGenConversationUpdated: () => () => undefined

integration-tests/workspace-production-bridge.ui.integration.dbtest.tsx

@@ -175,8 +175,17 @@ beforeEach(() => {
   }
 })
 
-afterEach(() => {
+afterEach(async () => {
   cleanup()
+  // Background passes (follow-up suggestions) wait out the foreground grace and may still be queued
+  // when a test ends; let them finish so the next test's model replies are its own.
+  const { modalityQueue } = await import('../src/main/modality-queue/queue')
+  const deadline = Date.now() + 20_000
+  for (;;) {
+    const { running, queued } = modalityQueue.getState()
+    if ((!running.length && !queued.length) || Date.now() > deadline) break
+    await new Promise((resolve) => setTimeout(resolve, 100))
+  }
   fake.reset()
 })
 
@@ -213,6 +222,11 @@ async function inTranscript(text: string | RegExp): Promise<HTMLElement> {
   )
 }
 
+/** Follow-up suggestions are background passes: they wait out the 3 s foreground grace after a reply,
+ *  and behind any background pass still queued. Tests that wait for several get a matching timeout. */
+const SUGGESTION_TEST_TIMEOUT_MS = 60_000
+const SUGGESTION_WAIT_MS = 15_000
+
 describe('production workspace bridge', () => {
   it('sends a rendered chat turn through preload, IPC, the model socket, and SQLite', async () => {
     fake.enqueue(
@@ -238,7 +252,7 @@ describe('production workspace bridge', () => {
         ['assistant', 'The production bridge persisted this answer.']
       ])
     })
-    expect(await screen.findByPlaceholderText('Check the saved answer (Tab)')).toBeTruthy()
+    expect(await screen.findByPlaceholderText('Check the saved answer (Tab)', undefined, { timeout: SUGGESTION_WAIT_MS })).toBeTruthy()
     expect(fake.requests).toHaveLength(2)
   })
 
@@ -325,10 +339,12 @@ describe('production workspace bridge', () => {
   }, 15_000)
 
   it('shows the real memory-chat failure instead of a fabricated answer', async () => {
-    fake.enqueue({
+    // An unavailable model is tried again once (chat fallback policy): it fails both times here.
+    const unavailable = {
       errorStatus: 503,
       errorBody: JSON.stringify({ error: { message: 'Memory model is unavailable.' } })
-    })
+    }
+    fake.enqueue(unavailable, unavailable)
     const user = userEvent.setup()
     renderChat()
 
@@ -389,15 +405,15 @@ describe('production workspace bridge', () => {
     fireEvent.change(composer, { target: { value: 'Start a long conversation' } })
     await user.click(screen.getByRole('button', { name: /^send$/i }))
     expect(await inTranscript(longAnswer)).toBeTruthy()
-    expect(await screen.findByPlaceholderText('Continue after the context fills (Tab)')).toBeTruthy()
+    expect(await screen.findByPlaceholderText('Continue after the context fills (Tab)', undefined, { timeout: SUGGESTION_WAIT_MS })).toBeTruthy()
 
     fireEvent.change(composer, { target: { value: 'Continue after the context fills' } })
     await user.click(screen.getByRole('button', { name: /^send$/i }))
     expect(await inTranscript('The next answer still works.')).toBeTruthy()
     expect(
       await screen.findByText('Compacted conversation to make room for more messages.')
     ).toBeTruthy()
-    expect(await screen.findByPlaceholderText('Review the shorter conversation (Tab)')).toBeTruthy()
+    expect(await screen.findByPlaceholderText('Review the shorter conversation (Tab)', undefined, { timeout: SUGGESTION_WAIT_MS })).toBeTruthy()
     const summaryRequest = JSON.stringify(fake.requests[2])
     expect(summaryRequest).toContain('## Facts and preferences')
     const nextModelRequest = JSON.stringify(fake.requests[3])
@@ -413,7 +429,7 @@ describe('production workspace bridge', () => {
     const notice = getRagMessages(conversation!.id).find(({ content }) => content === '_Compacted_')
     expect(notice).toBeTruthy()
     expect(JSON.parse(String(notice!.context))).toEqual({ notice: true, compactionSummary: summary })
-  })
+  }, SUGGESTION_TEST_TIMEOUT_MS)
 
   it('uses the same chat Thinking control for remote provider requests', async () => {
     const endpoint = `http://127.0.0.1:${fake.port}/v1`
@@ -436,7 +452,7 @@ describe('production workspace bridge', () => {
         fireEvent.change(composer, { target: { value: `Test ${provider} with Thinking on` } })
         await user.click(screen.getByRole('button', { name: /^send$/i }))
         expect(await inTranscript(`${provider} answered with Thinking on.`)).toBeTruthy()
-        expect(await screen.findByPlaceholderText('Try with Thinking off (Tab)')).toBeTruthy()
+        expect(await screen.findByPlaceholderText('Try with Thinking off (Tab)', undefined, { timeout: SUGGESTION_WAIT_MS })).toBeTruthy()
         const onRequest = fake.requests[0]
         if (provider === 'openrouter') {
           expect(onRequest?.reasoning).toEqual({ max_tokens: 1024 })
@@ -451,7 +467,7 @@ describe('production workspace bridge', () => {
         fireEvent.change(composer, { target: { value: `Test ${provider} with Thinking off` } })
         await user.click(screen.getByRole('button', { name: /^send$/i }))
         expect(await inTranscript(`${provider} answered with Thinking off.`)).toBeTruthy()
-        expect(await screen.findByPlaceholderText('Compare both answers (Tab)')).toBeTruthy()
+        expect(await screen.findByPlaceholderText('Compare both answers (Tab)', undefined, { timeout: SUGGESTION_WAIT_MS })).toBeTruthy()
         const offRequest = fake.requests[2]
         if (provider === 'openrouter') {
           expect(offRequest?.reasoning).toEqual({ effort: 'none' })
@@ -470,7 +486,7 @@ describe('production workspace bridge', () => {
       })
       await window.api.setLlmSettings({ reasoningBudget: 0 })
     }
-  }, 15_000)
+  }, SUGGESTION_TEST_TIMEOUT_MS)
 
   it('shows Gemini thinking and completes a signed reasoning tool round', async () => {
     const endpoint = `http://127.0.0.1:${fake.port}/v1`

knip.json

@@ -26,7 +26,6 @@
     "@offgrid/design",
     "better-sqlite3",
     "@types/better-sqlite3",
-    "pdf-parse",
     "kokoro-js",
     "ollama"
   ],