From 840750b40adff9dcd9eef6af5e7c8cb78dbbe1f1 Mon Sep 17 00:00:00 2001 From: Niclas Jost Date: Sun, 1 Nov 2026 17:34:29 +0100 Subject: [PATCH 1/5] Load chat screenshots on demand and refetch an open chat once per burst The desktop fetched every chat whole, with each screenshot as base64 (one chat was 19.5 MB of JSON vs 0.78 MB light), and refetched it on every message, queue, run and conversation event (three times when a turn ends). Desktop now asks for ?lite=images (screenshots as imageUrl, tool output whole) and says messages.images on the socket so pushed messages come without pictures too. Message events patch the cache and cancel an in-flight fetch instead of refetching; run and conversation events refetch the chat once per 250 ms burst. Reading messages no longer scans all of message_images, and a picture request reads its block with json_each instead of parsing the whole message. --- .../src/components/chat/message-blocks.tsx | 27 ++++++++---- apps/desktop/src/lib/api.ts | 2 +- apps/desktop/src/lib/realtime.ts | 35 +++++++++++----- packages/core/src/server/routes/chat.ts | 4 +- packages/core/src/server/ws.ts | 10 ++++- packages/core/src/services/conversations.ts | 42 +++++++++++++------ packages/core/src/services/liteMessage.ts | 12 ++++-- packages/core/test/conversations.test.ts | 23 ++++++++++ packages/shared/src/events.ts | 2 + 9 files changed, 119 insertions(+), 38 deletions(-) diff --git a/apps/desktop/src/components/chat/message-blocks.tsx b/apps/desktop/src/components/chat/message-blocks.tsx index aed34be8..62d36fea 100644 --- a/apps/desktop/src/components/chat/message-blocks.tsx +++ b/apps/desktop/src/components/chat/message-blocks.tsx @@ -32,6 +32,7 @@ import { useSmoothText } from "./use-smooth-text"; import { UserBubble } from "./user-bubble"; import { CopyButton } from "./copy-button"; import { Lightbox } from "./lightbox"; +import { useCoreFileUrl } from "./core-file"; import { PauseMarker } from "./pause"; import { QuestionCard, viewOfBlock } from "./question-card"; import { ArtifactCard } from "@/components/artifacts/artifact-panel"; @@ -441,6 +442,16 @@ function imageSrc(image: string): string { return `data:${mime};base64,${image}`; } +type Picture = Pick; + +const hasPicture = (b: Picture) => !!(b.image || b.imageUrl); + +function usePictureSrc(picture: Picture): string | null { + const data = useMemo(() => (picture.image ? imageSrc(picture.image) : null), [picture.image]); + const remote = useCoreFileUrl(data ? undefined : picture.imageUrl, true); + return data ?? remote.src; +} + function StreamingMarkdown({ text, className }: { text: string; className?: string }) { const reduce = useReduceAnimations(); const smooth = useSmoothText(text); @@ -608,7 +619,7 @@ function CompactToolGroup({ steps, ctx, streaming }: { steps: Step[]; ctx: ToolC const metas = tools.map((s) => describe(s.block.name, s.block.input, ctx)); const runningIdx = streaming ? tools.findIndex((s) => stepRunning(s.block, true)) : -1; const failed = tools.filter((s) => s.block.isError).length; - const images = tools.filter((s) => s.block.image).map((s) => s.block.image!); + const images = tools.filter((s) => hasPicture(s.block)).map((s) => s.block); const expanded = manual ?? (streaming && !easy); const single = tools.length === 1; @@ -651,7 +662,7 @@ function CompactToolGroup({ steps, ctx, streaming }: { steps: Step[]; ctx: ToolC {!expanded && images.length > 0 && ( {images.slice(-3).map((img, i) => ( - + ))} )} @@ -769,7 +780,7 @@ function ToolStep({ const body = ( <> - {block.image && } + {hasPicture(block) && } {easy ? : } ); @@ -793,10 +804,11 @@ function ToolStep({ ); } -function Screenshot({ image, alt }: { image: string; alt: string }) { +function Screenshot({ picture, alt }: { picture: Picture; alt: string }) { const { t } = useTranslation(["chat", "common"]); const [zoom, setZoom] = useState(false); - const src = imageSrc(image); + const src = usePictureSrc(picture); + if (!src) return
; return ( <>