diff --git a/packages/client/src/promise/generated/types.ts b/packages/client/src/promise/generated/types.ts index 738fb50180fa..a93cd8c37b84 100644 --- a/packages/client/src/promise/generated/types.ts +++ b/packages/client/src/promise/generated/types.ts @@ -2426,6 +2426,7 @@ export type ConfigEntry = todo?: boolean background_compaction?: boolean media_blob?: boolean + image_model?: string | { providerID: string; model: string; variant?: string } schedules?: boolean subagent_depth?: number location_idle_timeout?: string | false diff --git a/packages/core/src/config/normalize.ts b/packages/core/src/config/normalize.ts index 6e0e1dd588bb..434f45051dcd 100644 --- a/packages/core/src/config/normalize.ts +++ b/packages/core/src/config/normalize.ts @@ -495,6 +495,15 @@ function normalizeExperimental( ) if (value !== undefined) result.media_blob = value } + if (own(experimental, "image_model")) { + const value = decodeEncoded( + ConfigExperimental.Info.fields.image_model, + experimental.image_model, + ["experimental", "image_model"], + diagnostics, + ) + if (value !== undefined) result.image_model = value + } if (own(experimental, "schedules")) { const value = decodeEncoded( ConfigExperimental.Info.fields.schedules, diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index 7afc0f17f229..4eb0ab39d498 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -23,6 +23,12 @@ const Background = Schema.Struct({ agent: Schema.String, description: Schema.String, }), + Schema.Struct({ + kind: Schema.Literal("image"), + sessionID: SessionSchema.ID, + path: Schema.String, + model: Schema.String, + }), ]), status: Schema.Literals(["running", "completed", "error", "cancelled"]), output: Schema.optionalKey(Schema.String), diff --git a/packages/core/src/location-activity.ts b/packages/core/src/location-activity.ts index 7aa3ed68dc9a..c2416e966dd3 100644 --- a/packages/core/src/location-activity.ts +++ b/packages/core/src/location-activity.ts @@ -77,7 +77,7 @@ export function layer(options: { readonly timeToLive?: Duration.Input; readonly const owners = [ ...(yield* execution.active), ...(yield* jobs.runningBackground).flatMap((recovery) => - recovery.kind === "shell" ? [recovery.sessionID] : [recovery.childSessionID, recovery.parentSessionID], + recovery.kind === "subagent" ? [recovery.childSessionID, recovery.parentSessionID] : [recovery.sessionID], ), ] const occupied = new Set( diff --git a/packages/core/src/plugin/goal.ts b/packages/core/src/plugin/goal.ts index bd841405450d..67b3daff8ff5 100644 --- a/packages/core/src/plugin/goal.ts +++ b/packages/core/src/plugin/goal.ts @@ -145,9 +145,12 @@ export const Plugin = define({ if (goal?.status !== "active") return if (goal.boot !== boot()) return yield* settle(sessionID, goal, "paused", "The server restarted while the goal was active") - // The Session's own running subagent wakes it with its report, and the hook runs again then. A grandchild reports - // to its own parent, so it would never wake this Session. - if (background.some((work) => work.kind === "subagent" && work.sessionID === sessionID)) return + // The Session's own running subagent or image job wakes it with its notice, and the hook runs again then. A + // grandchild reports to its own parent, so it would never wake this Session. + if ( + background.some((work) => (work.kind === "subagent" || work.kind === "image") && work.sessionID === sessionID) + ) + return if (goal.rounds >= goal.maxRounds) return yield* settle(sessionID, goal, "paused", `Reached the limit of ${goal.maxRounds} rounds`) diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index 0b00caa0ccc4..00df9f767a1f 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -39,6 +39,7 @@ import { Environment } from "../environment/index.js" import { FileAccess } from "../file-access.js" import { FileMutation } from "../file-mutation.js" import { Formatter } from "../formatter.js" +import { Generate } from "../generate.js" import { Form } from "../form.js" import { FileSystem } from "../filesystem.js" import { LocationWatcherPolicy } from "../filesystem/location-watcher-policy.js" @@ -71,6 +72,7 @@ import { PatchTool } from "../tool/plugin/patch.js" import { EditTool } from "../tool/plugin/edit.js" import { GlobTool } from "../tool/plugin/glob.js" import { GrepTool } from "../tool/plugin/grep.js" +import { ImageGenerateTool } from "../tool/plugin/image-generate.js" import { McpResourceTools } from "../tool/plugin/mcp-resource.js" import { OpenCodeTools } from "../tool/plugin/opencode.js" import { QuestionTool } from "../tool/plugin/question.js" @@ -119,6 +121,7 @@ const services = [ FileAccess.Service, FileMutation.Service, Formatter.Service, + Generate.Service, LocationWatcherPolicy.Service, FileSystem.Service, FSUtil.Service, @@ -172,6 +175,7 @@ export const requirements = LayerNode.group([ FileAccess.node, FileMutation.node, Formatter.node, + Generate.node, LocationWatcherPolicy.node, FileSystem.node, FSUtil.node, @@ -236,6 +240,7 @@ const pre = [ EditTool.Plugin, GlobTool.Plugin, GrepTool.Plugin, + ImageGenerateTool.Plugin, OpenCodeTools.Plugin, McpResourceTools.Plugin, QuestionTool.Plugin, diff --git a/packages/core/src/session/background.ts b/packages/core/src/session/background.ts index 1a247a7afa64..df8b38bb0ec3 100644 --- a/packages/core/src/session/background.ts +++ b/packages/core/src/session/background.ts @@ -16,24 +16,26 @@ export const outstanding = Effect.fnUntraced(function* ( sessionID: SessionSchema.ID, ) { const owned = yield* Effect.filter(yield* jobs.runningBackground, (recovery) => - descends(store, recovery.kind === "shell" ? recovery.sessionID : recovery.parentSessionID, sessionID), - ) - return owned.map((recovery) => - recovery.kind === "shell" - ? { - kind: "shell" as const, - sessionID: recovery.sessionID, - shellID: recovery.shellID, - command: recovery.command, - } - : { - kind: "subagent" as const, - sessionID: recovery.parentSessionID, - childSessionID: recovery.childSessionID, - agent: recovery.agent, - description: recovery.description, - }, + descends(store, recovery.kind === "subagent" ? recovery.parentSessionID : recovery.sessionID, sessionID), ) + return owned.map((recovery) => { + if (recovery.kind === "shell") + return { + kind: "shell" as const, + sessionID: recovery.sessionID, + shellID: recovery.shellID, + command: recovery.command, + } + if (recovery.kind === "image") + return { kind: "image" as const, sessionID: recovery.sessionID, path: recovery.path, model: recovery.model } + return { + kind: "subagent" as const, + sessionID: recovery.parentSessionID, + childSessionID: recovery.childSessionID, + agent: recovery.agent, + description: recovery.description, + } + }) }) const descends = Effect.fnUntraced(function* ( diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index dde339301837..bf4aa65b35a8 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -135,6 +135,39 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + const recoverImage = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + ) { + const text = + background.status === "running" + ? `Image request for ${recovery.path} was cancelled because the server restarted.` + : background.status === "completed" + ? (background.output ?? `Image request finished. Saved image: ${recovery.path} (${recovery.model}).`) + : background.status === "error" + ? (background.error ?? `Image request failed for ${recovery.path}.`) + : `Image request for ${recovery.path} was cancelled.` + yield* sessions + .synthetic({ + id: background.notificationID, + sessionID: recovery.sessionID, + description: "Image generation", + text, + metadata: { + source: "image_generate", + jobID: background.id, + state: background.status === "running" ? "cancelled" : background.status, + path: recovery.path, + }, + resume: false, + }) + .pipe( + Effect.catchTag("Session.NotFoundError", () => Effect.void), + Effect.orDie, + ) + yield* jobs.completeBackground(background.notificationID) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -209,9 +242,9 @@ export const layer = (options?: Options) => Effect.fnUntraced(function* (background) { if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - yield* recovery.kind === "shell" - ? recoverShell(background, recovery) - : recoverSubagent(background, recovery, suspended) + if (recovery.kind === "shell") return yield* recoverShell(background, recovery) + if (recovery.kind === "image") return yield* recoverImage(background, recovery) + yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, ) diff --git a/packages/core/src/tool/plugin/image-generate.ts b/packages/core/src/tool/plugin/image-generate.ts new file mode 100644 index 000000000000..1cdaec7aaf89 --- /dev/null +++ b/packages/core/src/tool/plugin/image-generate.ts @@ -0,0 +1,387 @@ +export * as ImageGenerateTool from "./image-generate.js" + +import path from "path" +import { ToolFailure } from "@opencode/ai/effect" +import type { Context } from "@opencode/plugin/effect/plugin" +import type { Document, Entry } from "@opencode/schema/config" +import { Identifier } from "@opencode/schema/identifier" +import { Global } from "@opencode/util/global" +import { Duration, Effect, Schema, Scope, Stream } from "effect" +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http" +import { Config } from "../../config.js" +import { Environment } from "../../environment/index.js" +import { FileAccess } from "../../file-access.js" +import { Generate } from "../../generate.js" +import { Job } from "../../job.js" +import { Mime } from "../../mime.js" +import { Model } from "../../model.js" +import { Permission } from "../../permission.js" +import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" +import { SessionSchema } from "../../session/schema.js" +import { collectBoundedResponseBody } from "../http-body.js" + +export const name = "image_generate" +export const MAX_INPUT_BYTES = 20 * 1024 * 1024 +const DOWNLOAD_TIMEOUT = Duration.seconds(30) +const GENERATE_TIMEOUT = Duration.minutes(30) +const INPUT_TYPES = ["image/png", "image/jpeg", "image/webp", "image/gif"] +const EXTENSIONS: Record = { + "image/png": ".png", + "image/jpeg": ".jpg", + "image/webp": ".webp", + "image/gif": ".gif", +} + +export const description = + "Generate or edit an image. Pass images to edit them or to guide style and subject. To make several images, call the tool once per image with distinct prompts. Omit model to use the default. Runs in the background: the call returns right away and you are notified automatically when the image is saved or fails. DO NOT sleep, poll, or check whether the file exists while it runs; continue with other work or end your response. After the notice, use the read tool on the path to view the image and mention the path to the user. Each call costs money, so don't regenerate unless asked or the result is clearly wrong." + +export const Input = Schema.Struct({ + prompt: Schema.String.check(Schema.isMinLength(1)).annotate({ + description: + "What to create, or for edits, what to change. Be specific about subject, style, composition and any text that should appear in the image.", + }), + images: Schema.optionalKey( + Schema.Array(Schema.String.check(Schema.isMinLength(1))).check(Schema.isMinLength(1), Schema.isMaxLength(8)), + ).annotate({ + description: + "Images to edit or use as references, in order. Local file paths (relative to the session directory or absolute) or http(s) URLs. Omit to create a new image.", + }), + model: Schema.optionalKey(Schema.String).annotate({ + description: + 'NEVER set this unless the user explicitly asks for a particular image model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look it up with the models tool using type: "image", filtering to your own provider first.', + }), + path: Schema.optionalKey(Schema.String).annotate({ + description: + "Where to save the image, relative to the session directory or absolute. The extension is replaced with the image's real format, and an existing file is overwritten. Defaults to a new file in the session's temp folder.", + }), +}) + +export const Output = Schema.Struct({ + path: Schema.String.annotate({ description: "Where the image is saved, without its extension" }), + model: Schema.String.annotate({ description: "providerID/modelID" }), +}) + +export const Plugin = { + id: "opencode.tool.image-generate", + effect: Effect.fn("ImageGenerateTool.Plugin")(function* (ctx: Context) { + const access = yield* FileAccess.Service + const environment = yield* Environment.Service + const generate = yield* Generate.Service + const global = yield* Global.Service + const http = yield* HttpClient.HttpClient + const config = yield* Config.Service + const jobs = yield* Job.Service + const scope = yield* Scope.Scope + const sessions = yield* Session.Service + const models = yield* Model.Service + const permission = yield* Permission.Service + + const resolveModel = Effect.fn("ImageGenerateTool.resolveModel")(function* ( + input: string | undefined, + sessionID: SessionSchema.ID, + images: boolean, + ) { + const available = yield* models.available(["image"]) + if (!input) { + // A configured default is a preference: when it can't serve this call, pick automatically and say so. + const configured = configuredModel(yield* config.entries()) + const preferred = configured + ? available.find((model) => model.providerID === configured.providerID && model.id === configured.model) + : undefined + const variantMissing = + configured?.variant !== undefined && + preferred !== undefined && + !preferred.variants.some((variant) => variant.id === configured.variant) + if (preferred && !variantMissing && (!images || acceptsImages(preferred))) + return { model: preferred, variant: configured?.variant } + const providerID = (yield* ctx.session.get({ sessionID })).model?.providerID + const model = defaultModel(available, { providerID, images }) + if (!model) + return yield* new ToolFailure({ + message: images + ? "None of the available image models accept input images." + : "No image model is available.", + }) + if (!configured) return { model } + const ref = `${configured.providerID}/${configured.model}${configured.variant ? `#${configured.variant}` : ""}` + const reason = !preferred + ? "isn't available" + : variantMissing + ? `has no variant "${configured.variant}"` + : "doesn't accept input images" + return { model, note: ` ${ref} ${reason}; used ${model.providerID}/${model.id}.` } + } + const ref = yield* Effect.try({ + try: () => Model.Ref.parse(input), + catch: () => + new ToolFailure({ + message: `Invalid model "${input}". Use "providerID/modelID" or "providerID/modelID#variant".`, + }), + }) + const model = available.find((model) => model.providerID === ref.providerID && model.id === ref.id) + if (model === undefined) + return yield* new ToolFailure({ + message: `Model "${ref.providerID}/${ref.id}" is not an available image model. Use the models tool with type: "image" to see what is available.`, + }) + if (ref.variant !== undefined && !model.variants.some((variant) => variant.id === ref.variant)) + return yield* new ToolFailure({ + message: + model.variants.length === 0 + ? `Model "${ref.providerID}/${ref.id}" has no variants. Omit the variant.` + : `Variant "${ref.variant}" is not available for "${ref.providerID}/${ref.id}". Available: ${model.variants.map((variant) => variant.id).join(", ")}.`, + }) + if (images && !acceptsImages(model)) { + const capable = available.filter(acceptsImages).slice(0, 3) + return yield* new ToolFailure({ + message: + capable.length === 0 + ? `Model "${ref.providerID}/${ref.id}" does not accept input images, and no available image model does.` + : `Model "${ref.providerID}/${ref.id}" does not accept input images. Image models that do: ${capable.map((model) => `${model.providerID}/${model.id}`).join(", ")}.`, + }) + } + return { model, variant: ref.variant } + }) + + const readImage = Effect.fn("ImageGenerateTool.readImage")(function* ( + source: string, + context: FileAccess.Invocation, + fetchAllowed: (url: string) => ReturnType, + ) { + const bytes = yield* readBytes(source, context, fetchAllowed) + const mime = Mime.detect(bytes) + if (!INPUT_TYPES.includes(mime)) + return yield* new ToolFailure({ message: `${source} is not a PNG, JPEG, WebP or GIF image.` }) + return { data: Buffer.from(bytes).toString("base64"), mime, name: path.basename(source) } + }) + + const readBytes = Effect.fn("ImageGenerateTool.readBytes")(function* ( + source: string, + context: FileAccess.Invocation, + fetchAllowed: (url: string) => ReturnType, + ) { + if (/^https?:\/\//i.test(source)) { + // Downloads follow the same rule as the webfetch tool, so denying web access also covers image URLs. + yield* fetchAllowed(source) + return yield* Effect.gen(function* () { + const response = yield* http + .execute(HttpClientRequest.get(source)) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)) + return yield* collectBoundedResponseBody( + response, + MAX_INPUT_BYTES, + () => new ToolFailure({ message: `${source} is larger than 20 MiB.` }), + ) + }).pipe( + Effect.timeoutOrElse({ + duration: DOWNLOAD_TIMEOUT, + orElse: () => Effect.fail(new ToolFailure({ message: `Timed out downloading ${source}.` })), + }), + Effect.mapError((error) => + error instanceof ToolFailure ? error : new ToolFailure({ message: `Unable to download ${source}`, error }), + ), + ) + } + const target = yield* access.authorizeRead(source, context) + const info = yield* environment.files + .stat(target.absolute) + .pipe(Effect.catchTag("Environment.NotFound", () => new ToolFailure({ message: `File not found: ${source}` }))) + if (info.size > MAX_INPUT_BYTES) return yield* new ToolFailure({ message: `${source} is larger than 20 MiB.` }) + return (yield* environment.files.read(target.absolute)).bytes + }) + + const notifyWhenDone = Effect.fn("ImageGenerateTool.notifyWhenDone")( + function* (sessionID: SessionSchema.ID, id: string, notificationID: SessionMessage.ID, file: string) { + const info = (yield* jobs.wait({ id })).info + if (!info || info.status === "running") return + yield* sessions.synthetic({ + id: notificationID, + sessionID, + description: "Image generation", + text: + info.status === "completed" + ? (info.output ?? "") + : info.status === "error" + ? (info.error ?? `Image request failed for ${file}.`) + : `Image request for ${file} was cancelled.`, + metadata: { source: name, jobID: id, state: info.status, path: file }, + }) + yield* jobs.completeBackground(notificationID) + }, + Effect.forkIn(scope, { startImmediately: true }), + ) + + // Registered only while an image model is available, re-evaluated whenever the catalog changes. + // Subscribing before the first read keeps a change in between from being missed. + const wanted = models.available(["image"]).pipe(Effect.map((available) => available.length > 0)) + let enabled = false + const refresh = wanted.pipe( + Effect.flatMap((next) => { + if (next === enabled) return Effect.void + enabled = next + return ctx.tool.reload() + }), + ) + yield* ctx.event.subscribe({ types: ["model.updated"] }).pipe( + Stream.runForEach(() => refresh), + Effect.forkScoped({ startImmediately: true }), + ) + enabled = yield* wanted + + yield* ctx.tool + .transform((editor) => { + if (!enabled) return + editor.add({ + name, + options: { codemode: false }, + description, + input: Input, + output: Output, + execute: (input, context) => + Effect.gen(function* () { + const source = { type: "tool" as const, messageID: context.messageID, id: context.id } + const resolved = yield* resolveModel(input.model, context.sessionID, input.images !== undefined) + const model = resolved.model + const variant = resolved.variant + const ref = `${model.providerID}/${model.id}${variant ? `#${variant}` : ""}` + const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined + // The extension the image is saved with follows its real format, whatever the requested path ends in. + const stem = target + ? target.absolute.slice(0, target.absolute.length - path.extname(target.absolute).length) + : path.join(global.tmp, "images", context.sessionID, `image_${Identifier.create(false)}`) + + yield* permission.assert({ + action: name, + resources: [ref], + save: ["*"], + metadata: { ...input, model: ref }, + sessionID: context.sessionID, + agent: context.agent, + source, + }) + if (target) { + yield* access.authorizeExternal([target], context) + yield* permission.assert({ + action: "edit", + resources: [target.resource], + save: ["*"], + sessionID: context.sessionID, + agent: context.agent, + source, + }) + } + const images = input.images + ? yield* Effect.forEach(input.images, (image) => + readImage(image, context, (url) => + permission.assert({ + action: "webfetch", + resources: [url], + save: ["*"], + metadata: { url }, + sessionID: context.sessionID, + agent: context.agent, + source, + }), + ), + ) + : undefined + + // Everything that needs the user or can fail fast happened above; the paid call runs as a background job. + const run = Effect.gen(function* () { + const result = yield* generate + .image({ + model: Model.Ref.make({ + providerID: model.providerID, + id: model.id, + ...(variant ? { variant } : {}), + }), + prompt: input.prompt, + n: 1, + ...(images ? { images } : {}), + }) + .pipe( + Effect.timeoutOrElse({ + duration: GENERATE_TIMEOUT, + orElse: () => Effect.fail(new Error("Image generation timed out after 30 minutes.")), + }), + ) + const image = result.images[0] + if (!image) return yield* Effect.fail(new Error(`${ref} returned no image.`)) + const bytes = Buffer.from(image.data, "base64") + const file = stem + (EXTENSIONS[Mime.detect(bytes)] ?? ".png") + // The edit prompt covered the requested path. Replacing a different existing file needs its own + // approval; a new file only has to pass the deny rules. + if (target && file !== target.absolute) { + const actual = yield* access.resolve({ path: file, kind: "file" }) + const replaces = yield* environment.files.stat(file).pipe( + Effect.as(true), + Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)), + ) + const request = { + action: "edit", + resources: [actual.resource], + sessionID: context.sessionID, + agent: context.agent, + source, + } + yield* replaces ? permission.assert({ ...request, save: ["*"] }) : permission.assertNotDenied(request) + } + yield* environment.files.write(file, bytes) + return `Image request finished. Saved image: ${file} (${ref}).\nUse the read tool on this path to view the image.` + }).pipe( + // The job keeps this text, so a restart can still deliver the outcome. + Effect.mapError((error) => new Error(`Image request failed for ${stem}: ${error.message}`)), + ) + const id = `image_${Identifier.create(false)}` + const notificationID = SessionMessage.ID.create() + const job = yield* jobs.start({ + id, + notificationID, + type: name, + title: input.prompt, + metadata: { sessionID: context.sessionID, path: stem, model: ref }, + recovery: { kind: "image", sessionID: context.sessionID, path: stem, model: ref }, + run, + }) + yield* jobs.background(job.id) + yield* notifyWhenDone(context.sessionID, job.id, notificationID, stem) + return { + output: { path: stem, model: ref }, + content: [ + `Image request started in the background with ${ref}. It will be saved as ${stem} with the extension of its real format (.png, .jpg, .webp or .gif); the notice gives the exact path.${resolved.note ?? ""}`, + "You will be notified automatically when it finishes or fails.", + "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", + "Work on other tasks, or briefly tell the user what you started and end your response.", + ].join("\n"), + metadata: { path: stem, model: ref, jobID: job.id }, + } + }).pipe( + Effect.mapError((error) => + error instanceof ToolFailure ? error : new ToolFailure({ message: "Unable to generate image", error }), + ), + ), + }) + }) + .pipe(Effect.orDie) + }), +} + +/** The newest image model from the session's provider, otherwise the newest available one. */ +export function defaultModel( + available: readonly Model.Info[], + input: { readonly providerID?: string; readonly images: boolean }, +) { + const candidates = input.images ? available.filter(acceptsImages) : available + return candidates.find((model) => model.providerID === input.providerID) ?? candidates[0] +} + +// Read per field so a later config that sets other experimental options doesn't hide this one. +function configuredModel(entries: readonly Entry[]) { + return entries.findLast( + (entry): entry is Document => entry.type === "document" && entry.info.experimental?.image_model !== undefined, + )?.info.experimental?.image_model +} + +function acceptsImages(model: Model.Info) { + return model.capabilities.input.some((item) => item.startsWith("image")) +} diff --git a/packages/core/src/tool/plugin/opencode.ts b/packages/core/src/tool/plugin/opencode.ts index c5783f6dc0a0..d18bb45e41d1 100644 --- a/packages/core/src/tool/plugin/opencode.ts +++ b/packages/core/src/tool/plugin/opencode.ts @@ -50,6 +50,13 @@ export const ModelsInput = Schema.Struct({ provider: Schema.optionalKey(Schema.String).annotate({ description: "Provider ID or name to filter by. Try your own provider first.", }), + type: Schema.optionalKey(Schema.Literals(["chat", "image"])).annotate({ + description: [ + 'Which kind of model to list. Defaults to "chat".', + '"chat": a language model (LLM). Powers sessions, agents and subagents, and can call tools.', + '"image": generates or edits images from a prompt. It can\'t chat or call tools.', + ].join("\n"), + }), all: Schema.optionalKey(Schema.Boolean).annotate({ description: "Include older versions of each model family. By default only the newest version is listed.", }), @@ -219,7 +226,7 @@ export const Plugin = { draft.add({ name: "models", description: - "Search the models available to use. Use this to turn a model name the user mentions into an exact reference before running a subagent on it. Check your own provider first.", + "Search the models available to use. Use this to turn a model name the user mentions into an exact reference for a tool that takes a model. Check your own provider first.", input: ModelsInput, output: ModelsOutput, options: { namespace: "opencode", codemode: true }, @@ -231,7 +238,7 @@ export const Plugin = { const terms = input.query?.toLowerCase().split(/\s+/).filter(Boolean) ?? [] const names = new Map((yield* ctx.provider.list()).data.map((provider) => [provider.id, provider.name])) const provider = input.provider?.toLowerCase() - const matching = (yield* ctx.model.list()).data + const matching = (yield* ctx.model.list({ types: [input.type ?? "chat"] })).data .filter( (model) => provider === undefined || diff --git a/packages/core/src/tool/plugin/subagent.ts b/packages/core/src/tool/plugin/subagent.ts index bd7cfcc788e3..b3cc81f65150 100644 --- a/packages/core/src/tool/plugin/subagent.ts +++ b/packages/core/src/tool/plugin/subagent.ts @@ -36,7 +36,7 @@ export const Input = Schema.Struct({ prompt: Schema.String.annotate({ description: "The task for the subagent to perform" }), model: Schema.optionalKey(Schema.String).annotate({ description: - 'NEVER set this unless the user explicitly asks for a particular model or variant. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look the model up with the models tool, filtering to your own provider first.', + 'NEVER set this unless the user explicitly asks for a particular model or variant. It must be a chat model; image and decision models can\'t run subagents. The value is written as "providerID/modelID", or "providerID/modelID#variant" to include a variant. Do not guess the ID: look the model up with the models tool, filtering to your own provider first.', }), sessionID: Schema.optionalKey(SessionSchema.ID).annotate({ description: "Send a follow-up to an existing subagent, running or finished. Omit to start a new one.", @@ -83,10 +83,16 @@ export const Plugin = { const model = (yield* models.available()).find( (model) => model.providerID === ref.providerID && model.id === ref.id, ) - if (model === undefined) + if (model === undefined) { + const other = (yield* models.available(["image", "decision"])).find( + (model) => model.providerID === ref.providerID && model.id === ref.id, + ) return yield* new ToolFailure({ - message: `Model "${ref.providerID}/${ref.id}" is not available. Use the models tool to see what is available.`, + message: other + ? `Model "${ref.providerID}/${ref.id}" is ${other.type === "image" ? "an image" : "a decision"} model; subagents need a chat model. Use the models tool to find one.` + : `Model "${ref.providerID}/${ref.id}" is not available. Use the models tool to see what is available.`, }) + } if (ref.variant !== undefined && !model.variants.some((variant) => variant.id === ref.variant)) return yield* new ToolFailure({ message: diff --git a/packages/core/test/config/normalization.test.ts b/packages/core/test/config/normalization.test.ts index 887aa2557993..de47941e3185 100644 --- a/packages/core/test/config/normalization.test.ts +++ b/packages/core/test/config/normalization.test.ts @@ -545,6 +545,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + image_model: "openai/gpt-image-2", subagent_depth: 0, location_idle_timeout: "2 hours", policies: [{ action: "provider.use", resource: "custom", effect: "allow" }], @@ -556,6 +557,7 @@ describe("ConfigNormalize", () => { todo: true, background_compaction: true, media_blob: true, + image_model: { providerID: "openai", model: "gpt-image-2" }, subagent_depth: 0, location_idle_timeout: "7200000 millis", }, @@ -569,6 +571,19 @@ describe("ConfigNormalize", () => { }) }) + test("normalizes the image model setting", () => { + expect(normalized({ experimental: { image_model: "openai/gpt-image-2#hd" } }).encoded.experimental).toEqual({ + image_model: { providerID: "openai", model: "gpt-image-2", variant: "hd" }, + }) + for (const value of [false, true, "no-slash", 1, { model: "openai/gpt-image-2" }]) { + const result = normalized({ experimental: { image_model: value } }) + expect(result.encoded.experimental).toBeUndefined() + expect(result.diagnostics.map((item) => [item.kind, item.path])).toEqual([ + ["invalid", ["experimental", "image_model"]], + ]) + } + }) + test("never emits experimental policies", () => { expect( normalized({ experimental: { policies: [{ action: "provider.use", resource: "*", effect: "deny" }] } }).encoded, diff --git a/packages/core/test/plugin/fixture.ts b/packages/core/test/plugin/fixture.ts index fca86018b82f..c8f3a06a727c 100644 --- a/packages/core/test/plugin/fixture.ts +++ b/packages/core/test/plugin/fixture.ts @@ -4,6 +4,7 @@ import { Command } from "@opencode/core/command" import { Config } from "@opencode/core/config" import { Credential } from "@opencode/core/credential" import { Database } from "@opencode/core/database/database" +import { Environment } from "@opencode/core/environment/index" import { FileAccess } from "@opencode/core/file-access" import { LayerNodePlatform } from "@opencode/util/effect/app-node-platform" import { AppProcess } from "@opencode/util/process" @@ -11,9 +12,11 @@ import { LayerNode } from "@opencode/util/effect/layer-node" import { Bus } from "@opencode/core/bus" import { FileSystem } from "@opencode/core/filesystem" import { FSUtil } from "@opencode/util/fs-util" +import { Global } from "@opencode/util/global" import { Form } from "@opencode/core/form" import { Generate } from "@opencode/core/generate" import { Integration } from "@opencode/core/integration" +import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Location } from "@opencode/core/location" import { Mcp } from "@opencode/core/mcp/index" @@ -41,6 +44,7 @@ import { WorktreeStrategies } from "@opencode/core/worktree/strategies" import { Effect, Layer } from "effect" import { tempLocationLayer } from "../fixture/location" import { emptyMcpLayer } from "../fixture/mcp" +import { offlineModels } from "../fixture/models" const npmLayer = Layer.succeed( Npm.Service, @@ -78,7 +82,7 @@ const permissionLayer = Layer.succeed( }), ) -export const makePluginTestLayer = (permission = permissionLayer) => +export const makePluginTestLayer = (permission = permissionLayer, generate = generateLayer) => AppNodeBuilder.build( LayerNode.group([ AppProcess.node, @@ -89,7 +93,9 @@ export const makePluginTestLayer = (permission = permissionLayer) => Credential.node, Config.node, Database.node, + Environment.node, FileAccess.node, + Global.node, Bus.node, Form.node, Generate.node, @@ -101,6 +107,7 @@ export const makePluginTestLayer = (permission = permissionLayer) => Model.node, Command.node, Integration.node, + Job.node, KV.node, Mcp.node, Session.node, @@ -125,8 +132,9 @@ export const makePluginTestLayer = (permission = permissionLayer) => Npm.node.replace(npmLayer), Config.node.replace(configLayer), Mcp.node.replace(emptyMcpLayer), - Generate.node.replace(generateLayer), + Generate.node.replace(generate), Permission.node.replace(permission), + offlineModels, ], ).pipe(Layer.provideMerge(configLayer)) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index eaa135652993..4cd23d9af22a 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -540,6 +540,44 @@ describe("SessionRestart background recovery", () => { }), ) + it.effect("delivers an image outcome persisted before its notification, and cancels one still running", () => + Effect.gen(function* () { + const database = yield* Database.Service + const jobs = yield* Job.Service + const sessionID = Session.ID.make("ses_background_image") + yield* seedSessions(database, [sessionID]) + const recovery = (path: string) => ({ kind: "image" as const, sessionID, path, model: "test/painter" }) + yield* jobs.start({ + id: "image_done", + type: "image_generate", + recovery: recovery("/tmp/done.png"), + run: Effect.succeed("Image request finished. Saved image: /tmp/done.png (test/painter)."), + }) + yield* jobs.background("image_done") + yield* jobs.wait({ id: "image_done" }) + yield* jobs.start({ + id: "image_running", + type: "image_generate", + recovery: recovery("/tmp/running.png"), + run: Effect.never, + }) + yield* jobs.background("image_running") + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution(scope, () => Effect.void, undefined, restarted) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const inbox = yield* SessionInbox.list(database.db, sessionID) + expect(inbox.map((item) => (item.type === "synthetic" ? item.payload.text : undefined)).toSorted()).toEqual([ + "Image request finished. Saved image: /tmp/done.png (test/painter).", + "Image request for /tmp/running.png was cancelled because the server restarted.", + ]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + for (const delivered of [false, true]) { it.effect(`does not duplicate a shell notification already ${delivered ? "delivered" : "admitted"}`, () => Effect.gen(function* () { diff --git a/packages/core/test/tool-image-generate.test.ts b/packages/core/test/tool-image-generate.test.ts new file mode 100644 index 000000000000..3e4fc5fed5a5 --- /dev/null +++ b/packages/core/test/tool-image-generate.test.ts @@ -0,0 +1,515 @@ +import { describe, expect } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Bus } from "@opencode/core/bus" +import { Config } from "@opencode/core/config" +import { Generate } from "@opencode/core/generate" +import { Job } from "@opencode/core/job" +import { Location } from "@opencode/core/location" +import { Model } from "@opencode/core/model" +import { Permission } from "@opencode/core/permission" +import { Plugin } from "@opencode/core/plugin" +import { PluginHost } from "@opencode/core/plugin/host" +import { Provider } from "@opencode/core/provider" +import { Session } from "@opencode/core/session" +import { SessionEvent } from "@opencode/core/session/event" +import { Tool } from "@opencode/core/tool" +import { ImageGenerateTool } from "@opencode/core/tool/plugin/image-generate" +import { OpenCodeTools } from "@opencode/core/tool/plugin/opencode" +import { Document, Event, Info } from "@opencode/schema/config" +import { ConfigExperimental } from "@opencode/schema/config/experimental" +import type { ImageRequest, ImageResult } from "@opencode/schema/generate" +import { Global } from "@opencode/util/global" +import { Effect, Layer, Schedule, Stream } from "effect" +import { testEffect } from "./lib/effect" +import { permissionLayer } from "./lib/permission" +import { executeTool, toolDefinitions, toolIdentity } from "./lib/tool" +import { makePluginTestLayer } from "./plugin/fixture" + +const PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +const JPEG = + "/9j/4AAQSkZJRgABAQAASABIAAD/4QBMRXhpZgAATU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAAABaADAAQAAAABAAAAAwAAAAD/7QA4UGhvdG9zaG9wIDMuMAA4QklNBAQAAAAAAAA4QklNBCUAAAAAABDUHYzZjwCyBOmACZjs+EJ+/8AAEQgAAwAFAwEiAAIRAQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMCBAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAAAAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEHYXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX29/j5+v/bAEMAAgICAgICAwICAwUDAwMFBgUFBQUGCAYGBgYGCAoICAgICAgKCgoKCgoKCgwMDAwMDA4ODg4ODw8PDw8PDw8PD//bAEMBAgICBAQEBwQEBxALCQsQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEP/dAAQAAf/aAAwDAQACEQMRAD8A7iiiiv8AQA+XP//Z" + +const fixture = { + requests: [] as ImageRequest[], + assertions: [] as Permission.AssertInput[], + checked: [] as Permission.AssertInput[], + result: { images: [{ type: "image", data: PNG, mime: "image/png" }] } as ImageResult, + // Holds generation until the test releases it, to observe a running job. + gate: Effect.void as Effect.Effect, + failure: undefined as Generate.Error | undefined, + deny: undefined as string | undefined, +} + +const it = testEffect( + makePluginTestLayer( + permissionLayer({ + assertNotDenied: (input) => Effect.sync(() => void fixture.checked.push(input)), + assert: (input) => + Effect.sync(() => void fixture.assertions.push(input)).pipe( + Effect.andThen( + fixture.deny === input.action + ? Effect.fail( + new Permission.BlockedError({ rules: [], permission: input.action, resources: input.resources }), + ) + : Effect.void, + ), + ), + }), + Layer.succeed( + Generate.Service, + Generate.Service.of({ + text: () => Effect.succeed(""), + image: (input) => + Effect.sync(() => fixture.requests.push(input)).pipe( + Effect.andThen(Effect.suspend(() => fixture.gate)), + Effect.andThen( + Effect.suspend(() => (fixture.failure ? Effect.fail(fixture.failure) : Effect.succeed(fixture.result))), + ), + ), + decisions: () => Effect.die("unused"), + }), + ), + ), +) + +const imageModel = (providerID: string, id: string, update: (model: Model.MutableInfo) => void = () => {}) => + Effect.gen(function* () { + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => + editor.models.update(Provider.ID.make(providerID), Model.ID.make(id), (model) => { + model.type = "image" + update(model) + }), + ) + }) + +const setup = (providerID = "test") => + Effect.gen(function* () { + fixture.requests.length = 0 + fixture.assertions.length = 0 + fixture.checked.length = 0 + fixture.result = { images: [{ type: "image", data: PNG, mime: "image/png" }] } + fixture.gate = Effect.void + fixture.failure = undefined + fixture.deny = undefined + const plugins = yield* Plugin.Service + const sessions = yield* Session.Service + const location = yield* Location.Service + const host = yield* PluginHost.make(plugins) + yield* ImageGenerateTool.Plugin.effect(host) + yield* OpenCodeTools.Plugin.effect(host) + const session = yield* sessions.create({ + location: Location.Ref.make({ directory: location.directory }), + model: Model.Ref.make({ providerID: Provider.ID.make(providerID), id: Model.ID.make("chat") }), + }) + const registry = yield* Tool.Service + const run = (input: Record) => + executeTool(registry, { + sessionID: session.id, + ...toolIdentity, + call: { type: "tool-call", id: `call-${JSON.stringify(input)}`, name: ImageGenerateTool.name, input }, + }) + // Completion notices arrive as synthetic messages posted by the background job. + const notices: { text: string; metadata?: Record }[] = [] + const bus = yield* Bus.Service + yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.runForEach((event) => + Effect.sync(() => { + if (event.data.sessionID === session.id && event.data.item.type === "synthetic") + notices.push(event.data.item.payload) + }), + ), + Effect.forkScoped({ startImmediately: true }), + ) + const notice = (result: { readonly metadata?: Record }) => + Effect.sync(() => notices.find((item) => item.metadata?.jobID === result.metadata?.jobID)).pipe( + Effect.repeat({ until: (item) => item !== undefined, schedule: Schedule.spaced("5 millis") }), + Effect.timeout("10 seconds"), + Effect.map((item) => item?.text), + ) + return { session, run, notice, directory: location.directory } + }) + +const configure = (image_model: ConfigExperimental.Info["image_model"]) => + Effect.gen(function* () { + const config = yield* Config.Test + yield* config.setEntries([ + new Document({ + type: "document", + info: new Info({ experimental: new ConfigExperimental.Info({ image_model }) }), + }), + ]) + const bus = yield* Bus.Service + yield* bus.publish(Event.Updated, {}) + }) + +const configuredModel = (providerID: string, model: string) => ({ + providerID: Provider.ID.make(providerID), + model: Model.ID.make(model), +}) + +const registered = Tool.Service.use((registry) => toolDefinitions(registry)).pipe( + Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)), +) +// Catalog and config changes reach the tool through events, so wait for them instead of a fixed delay. +const until = (expected: boolean) => + registered.pipe( + Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }), + Effect.timeout("10 seconds"), + ) + +const text = (result: { readonly content?: ReadonlyArray }) => + result.content?.[0]?.type === "text" ? result.content[0].text : undefined +const failure = (result: { readonly error?: { readonly message: string } }) => result.error?.message + +describe("image_generate", () => { + it.live("is registered only while an image model is available", () => + Effect.gen(function* () { + yield* setup() + expect(yield* registered).toBe(false) + yield* imageModel("test", "painter") + expect(yield* until(true)).toBe(true) + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => editor.models.remove(Provider.ID.make("test"), Model.ID.make("painter"))) + expect(yield* until(false)).toBe(false) + }), + ) + + it.live("uses the configured model unless the agent picks one, and falls back with a note", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + yield* imageModel("other", "text-only", (model) => { + model.capabilities.input = ["text"] + }) + yield* configure(configuredModel("other", "text-only")) + const { run, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + // The configured model beats the session's provider, and an explicit model beats both. + const configured = yield* run({ prompt: "x" }) + expect(configured.output).toMatchObject({ model: "other/text-only" }) + expect(text(configured)).not.toContain("used") + expect((yield* run({ prompt: "x", model: "test/painter" })).output).toMatchObject({ model: "test/painter" }) + + // A configured model that can't serve the call is skipped, and the result says why. + const edit = yield* run({ prompt: "x", images: ["in.png"] }) + expect(edit.output).toMatchObject({ model: "test/painter" }) + expect(text(edit)).toContain("other/text-only doesn't accept input images; used test/painter.") + + yield* configure(configuredModel("gone", "model")) + const missing = yield* run({ prompt: "x" }) + expect(missing.output).toMatchObject({ model: "test/painter" }) + expect(text(missing)).toContain("gone/model isn't available; used test/painter.") + }), + ) + + it.live("reads input images, asks permission, and saves the result, overwriting an existing file", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run, notice, directory, session } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + const result = yield* run({ prompt: "make it blue", images: ["in.png"], path: "out.png" }) + const file = path.join(directory, "out.png") + const stem = path.join(directory, "out") + expect(result.output).toEqual({ path: stem, model: "test/painter" }) + expect(result.content).toEqual([ + { + type: "text", + text: [ + `Image request started in the background with test/painter. It will be saved as ${stem} with the extension of its real format (.png, .jpg, .webp or .gif); the notice gives the exact path.`, + "You will be notified automatically when it finishes or fails.", + "DO NOT sleep, poll, check whether the file exists, or request the same image again while it runs.", + "Work on other tasks, or briefly tell the user what you started and end your response.", + ].join("\n"), + }, + ]) + expect(yield* notice(result)).toBe( + `Image request finished. Saved image: ${file} (test/painter).\nUse the read tool on this path to view the image.`, + ) + expect((yield* Effect.promise(() => fs.readFile(file))).toString("base64")).toBe(PNG) + expect(fixture.requests).toEqual([ + { + model: Model.Ref.make({ providerID: Provider.ID.make("test"), id: Model.ID.make("painter") }), + prompt: "make it blue", + n: 1, + images: [{ data: PNG, mime: "image/png", name: "in.png" }], + }, + ]) + expect(fixture.assertions.map((item) => [item.action, item.resources])).toEqual([ + ["image_generate", ["test/painter"]], + ["edit", ["out.png"]], + ["read", ["in.png"]], + ]) + expect(fixture.assertions[0]?.sessionID).toBe(session.id) + + // Running again with the same path replaces the file, like the write tool. + const again = yield* run({ prompt: "again", path: "out.png" }) + expect(again.status).toBe("completed") + yield* notice(again) + expect(fixture.requests).toHaveLength(2) + }), + ) + + it.live("saves with the extension of the image's real format", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, notice, session, directory } = yield* setup() + const global = yield* Global.Service + fixture.result = { images: [{ type: "image", data: JPEG, mime: "image/jpeg" }] } + + // The default path lives in the session temp folder and takes the real extension. + const result = yield* run({ prompt: "a cat" }) + expect(path.dirname(result.output.path)).toBe(path.join(global.tmp, "images", session.id)) + const saved = `${result.output.path}.jpg` + expect(yield* notice(result)).toBe( + `Image request finished. Saved image: ${saved} (test/painter).\nUse the read tool on this path to view the image.`, + ) + expect(yield* Effect.promise(() => Bun.file(saved).exists())).toBe(true) + + // An explicit path's extension is replaced too, and the notice gives the real path. + const explicit = yield* run({ prompt: "a cat", path: "cat.png" }) + expect(yield* notice(explicit)).toContain(`Saved image: ${path.join(directory, "cat.jpg")} (test/painter).`) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "cat.jpg")).exists())).toBe(true) + // The edit prompt covered cat.png; the file actually written is still checked against deny rules. + expect(fixture.assertions.filter((item) => item.action === "edit").map((item) => item.resources)).toEqual([ + ["cat.png"], + ]) + expect(fixture.checked.map((item) => [item.action, item.resources])).toEqual([["edit", ["cat.jpg"]]]) + + // Replacing an existing file under the real extension asks for its own edit approval. + fixture.assertions.length = 0 + fixture.checked.length = 0 + const again = yield* run({ prompt: "a cat", path: "cat.png" }) + yield* notice(again) + expect(fixture.assertions.filter((item) => item.action === "edit").map((item) => item.resources)).toEqual([ + ["cat.png"], + ["cat.jpg"], + ]) + expect(fixture.checked).toEqual([]) + }), + ) + + it.live("reports a failed generation in the completion notice", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, notice, directory } = yield* setup() + fixture.failure = new Generate.ProviderRequestError({ + providerID: "test", + reason: "content_filter", + message: "The prompt was rejected", + }) + const result = yield* run({ prompt: "x", path: "out.png" }) + expect(yield* notice(result)).toBe( + `Image request failed for ${path.join(directory, "out")}: The prompt was rejected`, + ) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) + }), + ) + + it.live("cancels a running generation through its job", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, notice, directory } = yield* setup() + fixture.gate = Effect.never + const result = yield* run({ prompt: "x", path: "out.png" }) + const jobs = yield* Job.Service + const id = String(result.metadata?.jobID) + expect((yield* jobs.get(id))?.status).toBe("running") + yield* jobs.cancel(id) + expect(yield* notice(result)).toBe(`Image request for ${path.join(directory, "out")} was cancelled.`) + expect(yield* Effect.promise(() => Bun.file(path.join(directory, "out.png")).exists())).toBe(false) + }), + ) + + it.live("starts no job when a permission is denied", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run } = yield* setup() + fixture.deny = "edit" + const result = yield* run({ prompt: "x", path: "out.png" }) + expect(result.status).toBe("error") + expect(fixture.assertions.map((item) => item.action)).toEqual(["image_generate", "edit"]) + expect(fixture.requests).toEqual([]) + }), + ) + + it.live("validates an explicit model and its support for input images", () => + Effect.gen(function* () { + yield* imageModel("test", "text-only", (model) => { + model.capabilities.input = ["text"] + }) + yield* imageModel("other", "editor", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run, notice, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64"))) + + expect(failure(yield* run({ prompt: "x", model: "test/missing" }))).toBe( + 'Model "test/missing" is not an available image model. Use the models tool with type: "image" to see what is available.', + ) + expect(failure(yield* run({ prompt: "x", model: "test/text-only", images: ["in.png"] }))).toBe( + 'Model "test/text-only" does not accept input images. Image models that do: other/editor.', + ) + // Without an explicit model, input images steer the default to a model that accepts them. + const steered = yield* run({ prompt: "x", images: ["in.png"] }) + expect(steered.output).toMatchObject({ model: "other/editor" }) + const explicit = yield* run({ prompt: "x", model: "test/text-only" }) + expect(explicit.output).toMatchObject({ model: "test/text-only" }) + yield* notice(steered) + yield* notice(explicit) + expect(fixture.requests).toHaveLength(2) + }), + ) + + it.live("passes a model variant through, from the input or from config", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.variants = [{ id: Model.VariantID.make("hd"), settings: { quality: "high" } }] + }) + yield* imageModel("test", "plain") + const { run, notice } = yield* setup() + + const explicit = yield* run({ prompt: "x", model: "test/painter#hd" }) + expect(explicit.output).toMatchObject({ model: "test/painter#hd" }) + yield* notice(explicit) + expect(fixture.requests.at(-1)?.model).toEqual( + Model.Ref.make({ + providerID: Provider.ID.make("test"), + id: Model.ID.make("painter"), + variant: Model.VariantID.make("hd"), + }), + ) + expect(failure(yield* run({ prompt: "x", model: "test/painter#bogus" }))).toBe( + 'Variant "bogus" is not available for "test/painter". Available: hd.', + ) + expect(failure(yield* run({ prompt: "x", model: "test/plain#hd" }))).toBe( + 'Model "test/plain" has no variants. Omit the variant.', + ) + + yield* configure({ + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("hd"), + }) + const configured = yield* run({ prompt: "x" }) + expect(configured.output).toMatchObject({ model: "test/painter#hd" }) + yield* notice(configured) + expect(fixture.requests.at(-1)?.model.variant).toBe(Model.VariantID.make("hd")) + + yield* configure({ + providerID: Provider.ID.make("test"), + model: Model.ID.make("painter"), + variant: Model.VariantID.make("bogus"), + }) + const fallback = yield* run({ prompt: "x" }) + expect(text(fallback)).toContain('test/painter#bogus has no variant "bogus"; used ') + yield* notice(fallback) + expect(fixture.requests.at(-1)?.model.variant).toBeUndefined() + }), + ) + + it.live("asks webfetch permission before downloading an image URL", () => + Effect.gen(function* () { + yield* imageModel("test", "painter", (model) => { + model.capabilities.input = ["text", "image"] + }) + const { run } = yield* setup() + fixture.deny = "webfetch" + const result = yield* run({ prompt: "x", images: ["https://example.invalid/cat.png"] }) + expect(result.status).toBe("error") + expect(fixture.assertions.map((item) => [item.action, item.resources])).toEqual([ + ["image_generate", ["test/painter"]], + ["webfetch", ["https://example.invalid/cat.png"]], + ]) + expect(fixture.requests).toEqual([]) + }), + ) + + it.live("rejects input files that are not images", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const { run, directory } = yield* setup() + yield* Effect.promise(() => fs.writeFile(path.join(directory, "notes.txt"), "hello")) + expect(failure(yield* run({ prompt: "x", images: ["notes.txt"] }))).toBe( + "notes.txt is not a PNG, JPEG, WebP or GIF image.", + ) + expect(failure(yield* run({ prompt: "x", images: ["missing.png"] }))).toBe("File not found: missing.png") + expect(fixture.requests).toEqual([]) + }), + ) + + it.effect("lists image models through the models tool type filter", () => + Effect.gen(function* () { + yield* imageModel("test", "painter") + const catalog = yield* Provider.Service + yield* catalog.transform((editor) => + editor.models.update(Provider.ID.make("test"), Model.ID.make("chat"), () => {}), + ) + const { session } = yield* setup() + const registry = yield* Tool.Service + const models = (input: Record) => + executeTool(registry, { + sessionID: session.id, + ...toolIdentity, + call: { + type: "tool-call", + id: `models-${JSON.stringify(input)}`, + name: "execute", + input: { code: `return await tools.opencode.models(${JSON.stringify(input)})` }, + }, + }).pipe( + Effect.map((result) => + (JSON.parse(text(result) ?? "") as { providers: { models: { id: string }[] }[] }).providers.flatMap( + (provider) => provider.models.map((model) => model.id), + ), + ), + ) + expect(yield* models({ type: "image" })).toEqual(["test/painter"]) + expect(yield* models({})).not.toContain("test/painter") + expect(yield* models({})).toContain("test/chat") + }), + ) +}) + +describe("defaultModel", () => { + const info = (providerID: string, id: string, family?: string, input = ["text", "image"]) => ({ + ...Model.Info.default(Provider.ID.make(providerID), Model.ID.make(id)), + type: "image", + ...(family ? { family: Model.Family.make(family) } : {}), + capabilities: { tools: false, input, output: ["image"] }, + }) + const pick = (models: ReturnType[], providerID?: string, images = false) => { + const model = ImageGenerateTool.defaultModel(models, { providerID, images }) + return model && `${model.providerID}/${model.id}` + } + + it.effect("prefers the session's provider, then the newest", () => + Effect.sync(() => { + const newest = info("fresh", "newest") + const older = info("openai", "gpt-image-2", "gpt-image") + const own = info("mine", "own-image") + const all = [newest, older, own] + + expect(pick(all, "mine")).toBe("mine/own-image") + expect(pick(all, "elsewhere")).toBe("fresh/newest") + expect(pick(all)).toBe("fresh/newest") + expect(pick([])).toBeUndefined() + }), + ) + + it.effect("only considers models that accept input images when images are passed", () => + Effect.sync(() => { + const textOnly = info("mine", "text-only", "gpt-image", ["text"]) + const editor = info("other", "editor") + expect(pick([textOnly, editor], "mine", true)).toBe("other/editor") + expect(pick([textOnly], "mine", true)).toBeUndefined() + expect(pick([textOnly, editor], "mine", false)).toBe("mine/text-only") + }), + ) +}) diff --git a/packages/core/test/tool-subagent.test.ts b/packages/core/test/tool-subagent.test.ts index bc579b0d121c..7cd56541bd16 100644 --- a/packages/core/test/tool-subagent.test.ts +++ b/packages/core/test/tool-subagent.test.ts @@ -184,6 +184,12 @@ const withSubagent = (location: Location.Ref) => model.variants = [{ id: Model.VariantID.make("fast") }] }) editor.models.update(Provider.ID.make("test"), Model.ID.make("plain"), () => {}) + editor.models.update(Provider.ID.make("test"), Model.ID.make("painter"), (model) => { + model.type = "image" + }) + editor.models.update(Provider.ID.make("test"), Model.ID.make("classifier"), (model) => { + model.type = "decision" + }) }), ).pipe(Effect.provide(locations.get(location))) yield* Agent.Service.use((agents) => @@ -774,6 +780,14 @@ describe("SubagentTool", () => { ["test/missing", 'Model "test/missing" is not available. Use the models tool to see what is available.'], ["test/override#slow", 'Variant "slow" is not available for "test/override". Available: fast.'], ["test/plain#high", 'Model "test/plain" has no variants. Omit the variant.'], + [ + "test/painter", + 'Model "test/painter" is an image model; subagents need a chat model. Use the models tool to find one.', + ], + [ + "test/classifier", + 'Model "test/classifier" is a decision model; subagents need a chat model. Use the models tool to find one.', + ], ] as const for (const [model, message] of failures) { expect(yield* call(`call-${model}`, { model })).toEqual({ diff --git a/packages/plugin/src/effect/session.ts b/packages/plugin/src/effect/session.ts index 920ecf2049d0..c5f04bc673b6 100644 --- a/packages/plugin/src/effect/session.ts +++ b/packages/plugin/src/effect/session.ts @@ -154,6 +154,12 @@ export type SessionIdleBackground = readonly agent: string readonly description: string } + | { + readonly kind: "image" + readonly sessionID: Session.ID + readonly path: string + readonly model: string + } /** * Runs when a Session finishes model work and has no pending input, just before it goes idle. diff --git a/packages/plugin/src/promise/session.ts b/packages/plugin/src/promise/session.ts index 743bd8f7d6d1..1621ce8b30d2 100644 --- a/packages/plugin/src/promise/session.ts +++ b/packages/plugin/src/promise/session.ts @@ -154,6 +154,12 @@ export type SessionIdleBackground = readonly agent: string readonly description: string } + | { + readonly kind: "image" + readonly sessionID: Session.ID + readonly path: string + readonly model: string + } /** * Runs when a Session finishes model work and has no pending input, just before it goes idle. diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index cdd955603890..5abbbeaed084 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -15790,6 +15790,33 @@ "media_blob": { "type": "boolean" }, + "image_model": { + "anyOf": [ + { + "type": "string", + "pattern": "^[^/#]+\\/[^#]+(?:#[^#]+)?$" + }, + { + "type": "object", + "properties": { + "providerID": { + "type": "string", + "pattern": "^[^/#]+$" + }, + "model": { + "type": "string", + "pattern": "^[^#]+$" + }, + "variant": { + "type": "string", + "pattern": "^[^#]+$" + } + }, + "required": ["providerID", "model"], + "additionalProperties": false + } + ] + }, "schedules": { "type": "boolean" }, diff --git a/packages/schema/src/config/experimental.ts b/packages/schema/src/config/experimental.ts index 2541219fa5e9..ec463c856d3b 100644 --- a/packages/schema/src/config/experimental.ts +++ b/packages/schema/src/config/experimental.ts @@ -2,6 +2,7 @@ export * as ConfigExperimental from "./experimental.js" import { Schema } from "effect" import { NonNegativeInt, optional } from "../schema.js" +import { ConfigModel } from "./model.js" import { ConfigPolicy } from "./policy.js" export class Info extends Schema.Class("ConfigExperimental.Info")({ @@ -19,6 +20,10 @@ export class Info extends Schema.Class("ConfigExperimental.Info")({ description: "Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.", }), + image_model: ConfigModel.Selection.pipe(optional).annotate({ + description: + "Default model for the image_generate tool when the agent doesn't choose one, as providerID/modelID. Falls back to automatic selection when it isn't available.", + }), schedules: Schema.Boolean.pipe(optional).annotate({ description: "Enable experimental scheduled tasks that wake a session with a prompt. Defaults to false.", }),