Skip to content

Commit cdc3b35

Browse files
committed
feat(core): add experimental.image_generation config
1 parent e3ba3dc commit cdc3b35

7 files changed

Lines changed: 197 additions & 26 deletions

File tree

‎packages/client/src/promise/generated/types.ts‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/core/src/config/normalize.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,15 @@ function normalizeExperimental(
495495
)
496496
if (value !== undefined) result.media_blob = value
497497
}
498+
if (own(experimental, "image_generation")) {
499+
const value = decodeEncoded(
500+
ConfigExperimental.Info.fields.image_generation,
501+
experimental.image_generation,
502+
["experimental", "image_generation"],
503+
diagnostics,
504+
)
505+
if (value !== undefined) result.image_generation = value
506+
}
498507
if (own(experimental, "schedules")) {
499508
const value = decodeEncoded(
500509
ConfigExperimental.Info.fields.schedules,

‎packages/core/src/tool/plugin/image-generate.ts‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ export * as ImageGenerateTool from "./image-generate.js"
33
import path from "path"
44
import { ToolFailure } from "@opencode/ai/effect"
55
import type { Context } from "@opencode/plugin/effect/plugin"
6+
import type { Document, Entry } from "@opencode/schema/config"
67
import { Identifier } from "@opencode/schema/identifier"
78
import type { Session } from "@opencode/schema/session"
89
import { Global } from "@opencode/util/global"
910
import { Duration, Effect, Schema, Stream } from "effect"
1011
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http"
12+
import { Config } from "../../config.js"
1113
import { Environment } from "../../environment/index.js"
1214
import { FileAccess } from "../../file-access.js"
1315
import { Generate } from "../../generate.js"
@@ -67,6 +69,7 @@ export const Plugin = {
6769
const generate = yield* Generate.Service
6870
const global = yield* Global.Service
6971
const http = yield* HttpClient.HttpClient
72+
const config = yield* Config.Service
7073
const models = yield* Model.Service
7174
const permission = yield* Permission.Service
7275

@@ -77,12 +80,25 @@ export const Plugin = {
7780
) {
7881
const available = yield* models.available(["image"])
7982
if (!input) {
83+
// A configured default is a preference: when it can't serve this call, pick automatically and say so.
84+
const settings = settingsFrom(yield* config.entries())
85+
const configured = settings ? settings.model : undefined
86+
const preferred = configured
87+
? available.find((model) => model.providerID === configured.providerID && model.id === configured.model)
88+
: undefined
89+
if (preferred && (!images || acceptsImages(preferred))) return { model: preferred }
8090
const providerID = (yield* ctx.session.get({ sessionID })).model?.providerID
8191
const model = defaultModel(available, { providerID, images })
82-
if (model) return model
83-
return yield* new ToolFailure({
84-
message: images ? "None of the available image models accept input images." : "No image model is available.",
85-
})
92+
if (!model)
93+
return yield* new ToolFailure({
94+
message: images
95+
? "None of the available image models accept input images."
96+
: "No image model is available.",
97+
})
98+
if (!configured) return { model }
99+
const ref = `${configured.providerID}/${configured.model}`
100+
const reason = preferred ? "doesn't accept input images" : "isn't available"
101+
return { model, note: ` ${ref} ${reason}; used ${model.providerID}/${model.id}.` }
86102
}
87103
const ref = yield* Effect.try({
88104
try: () => Model.Ref.parse(input),
@@ -102,7 +118,7 @@ export const Plugin = {
102118
: `Model "${ref.providerID}/${ref.id}" does not accept input images. Image models that do: ${capable.map((model) => `${model.providerID}/${model.id}`).join(", ")}.`,
103119
})
104120
}
105-
return model
121+
return { model }
106122
})
107123

108124
const readImage = Effect.fn("ImageGenerateTool.readImage")(function* (
@@ -161,21 +177,24 @@ export const Plugin = {
161177
Effect.catchTag("Environment.NotFound", () => Effect.succeed(false)),
162178
)
163179

164-
// Registered only while an image model is available, and re-evaluated whenever the model catalog changes.
165-
// Subscribing before the first read keeps a change in between from being missed.
180+
// Registered only while an image model is available and config doesn't turn the tool off, re-evaluated
181+
// whenever either changes. Subscribing before the first read keeps a change in between from being missed.
182+
const wanted = Effect.all([config.entries(), models.available(["image"])]).pipe(
183+
Effect.map(([entries, available]) => settingsFrom(entries) !== false && available.length > 0),
184+
)
166185
let enabled = false
167-
const refresh = models.available(["image"]).pipe(
168-
Effect.flatMap((available) => {
169-
if (available.length > 0 === enabled) return Effect.void
170-
enabled = available.length > 0
186+
const refresh = wanted.pipe(
187+
Effect.flatMap((next) => {
188+
if (next === enabled) return Effect.void
189+
enabled = next
171190
return ctx.tool.reload()
172191
}),
173192
)
174-
yield* ctx.event.subscribe({ types: ["model.updated"] }).pipe(
193+
yield* ctx.event.subscribe({ types: ["model.updated", "config.updated"] }).pipe(
175194
Stream.runForEach(() => refresh),
176195
Effect.forkScoped({ startImmediately: true }),
177196
)
178-
enabled = (yield* models.available(["image"])).length > 0
197+
enabled = yield* wanted
179198

180199
yield* ctx.tool
181200
.transform((editor) => {
@@ -189,7 +208,8 @@ export const Plugin = {
189208
execute: (input, context) =>
190209
Effect.gen(function* () {
191210
const source = { type: "tool" as const, messageID: context.messageID, id: context.id }
192-
const model = yield* resolveModel(input.model, context.sessionID, input.images !== undefined)
211+
const resolved = yield* resolveModel(input.model, context.sessionID, input.images !== undefined)
212+
const model = resolved.model
193213
const ref = `${model.providerID}/${model.id}`
194214
const target = input.path ? yield* access.resolve({ path: input.path, kind: "file" }) : undefined
195215
const requested =
@@ -250,7 +270,9 @@ export const Plugin = {
250270
if (yield* exists(file))
251271
return yield* new ToolFailure({ message: `${file} already exists. Choose a new path.` })
252272
yield* environment.files.write(file, bytes)
253-
const note = mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.`
273+
const note =
274+
(resolved.note ?? "") +
275+
(mime === "image/png" ? "" : ` The model returned ${mime}, so the file is in that format.`)
254276
return {
255277
output: { path: file, model: ref },
256278
content: [
@@ -284,6 +306,13 @@ export function defaultModel(
284306
return PREFERRED_FAMILIES.flatMap((family) => pool.find((model) => model.family === family) ?? [])[0] ?? pool[0]
285307
}
286308

309+
// Read per field so a later config that sets other experimental options doesn't hide this one.
310+
function settingsFrom(entries: readonly Entry[]) {
311+
return entries.findLast(
312+
(entry): entry is Document => entry.type === "document" && entry.info.experimental?.image_generation !== undefined,
313+
)?.info.experimental?.image_generation
314+
}
315+
287316
function acceptsImages(model: Model.Info) {
288317
return model.capabilities.input.some((item) => item.startsWith("image"))
289318
}

‎packages/core/test/config/normalization.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,7 @@ describe("ConfigNormalize", () => {
545545
todo: true,
546546
background_compaction: true,
547547
media_blob: true,
548+
image_generation: { model: "openai/gpt-image-2" },
548549
subagent_depth: 0,
549550
location_idle_timeout: "2 hours",
550551
policies: [{ action: "provider.use", resource: "custom", effect: "allow" }],
@@ -556,6 +557,7 @@ describe("ConfigNormalize", () => {
556557
todo: true,
557558
background_compaction: true,
558559
media_blob: true,
560+
image_generation: { model: { providerID: "openai", model: "gpt-image-2" } },
559561
subagent_depth: 0,
560562
location_idle_timeout: "7200000 millis",
561563
},
@@ -569,6 +571,22 @@ describe("ConfigNormalize", () => {
569571
})
570572
})
571573

574+
test("normalizes the image generation setting", () => {
575+
expect(normalized({ experimental: { image_generation: false } }).encoded.experimental).toEqual({
576+
image_generation: false,
577+
})
578+
expect(normalized({ experimental: { image_generation: {} } }).encoded.experimental).toEqual({
579+
image_generation: {},
580+
})
581+
for (const value of [true, { model: "no-slash" }, { model: 1 }, "openai/gpt-image-2"]) {
582+
const result = normalized({ experimental: { image_generation: value } })
583+
expect(result.encoded.experimental).toBeUndefined()
584+
expect(result.diagnostics.map((item) => [item.kind, item.path])).toEqual([
585+
["invalid", ["experimental", "image_generation"]],
586+
])
587+
}
588+
})
589+
572590
test("never emits experimental policies", () => {
573591
expect(
574592
normalized({ experimental: { policies: [{ action: "provider.use", resource: "*", effect: "deny" }] } }).encoded,

‎packages/core/test/tool-image-generate.test.ts‎

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { describe, expect } from "bun:test"
22
import fs from "fs/promises"
33
import path from "path"
4+
import { Bus } from "@opencode/core/bus"
5+
import { Config } from "@opencode/core/config"
46
import { Generate } from "@opencode/core/generate"
57
import { Location } from "@opencode/core/location"
68
import { Model } from "@opencode/core/model"
@@ -12,6 +14,8 @@ import { Session } from "@opencode/core/session"
1214
import { Tool } from "@opencode/core/tool"
1315
import { ImageGenerateTool } from "@opencode/core/tool/plugin/image-generate"
1416
import { OpenCodeTools } from "@opencode/core/tool/plugin/opencode"
17+
import { Document, Event, Info } from "@opencode/schema/config"
18+
import { ConfigExperimental } from "@opencode/schema/config/experimental"
1519
import type { ImageRequest, ImageResult } from "@opencode/schema/generate"
1620
import { Global } from "@opencode/util/global"
1721
import { Effect, Layer, Schedule } from "effect"
@@ -82,6 +86,33 @@ const setup = (providerID = "test") =>
8286
return { session, run, directory: location.directory }
8387
})
8488

89+
const configure = (image_generation: ConfigExperimental.Info["image_generation"]) =>
90+
Effect.gen(function* () {
91+
const config = yield* Config.Test
92+
yield* config.setEntries([
93+
new Document({
94+
type: "document",
95+
info: new Info({ experimental: new ConfigExperimental.Info({ image_generation }) }),
96+
}),
97+
])
98+
const bus = yield* Bus.Service
99+
yield* bus.publish(Event.Updated, {})
100+
})
101+
102+
const configuredModel = (providerID: string, model: string) => ({
103+
model: { providerID: Provider.ID.make(providerID), model: Model.ID.make(model) },
104+
})
105+
106+
const registered = Tool.Service.use((registry) => toolDefinitions(registry)).pipe(
107+
Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)),
108+
)
109+
// Catalog and config changes reach the tool through events, so wait for them instead of a fixed delay.
110+
const until = (expected: boolean) =>
111+
registered.pipe(
112+
Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }),
113+
Effect.timeout("10 seconds"),
114+
)
115+
85116
const text = (result: { readonly content?: ReadonlyArray<Tool.Content> }) =>
86117
result.content?.[0]?.type === "text" ? result.content[0].text : undefined
87118
const failure = (result: { readonly error?: { readonly message: string } }) => result.error?.message
@@ -90,17 +121,6 @@ describe("image_generate", () => {
90121
it.live("is registered only while an image model is available", () =>
91122
Effect.gen(function* () {
92123
yield* setup()
93-
const registry = yield* Tool.Service
94-
const registered = toolDefinitions(registry).pipe(
95-
Effect.map((tools) => tools.some((tool) => tool.name === ImageGenerateTool.name)),
96-
)
97-
// The catalog change reaches the tool through model.updated, so wait for it instead of a fixed delay.
98-
const until = (expected: boolean) =>
99-
registered.pipe(
100-
Effect.repeat({ until: (value) => value === expected, schedule: Schedule.spaced("5 millis") }),
101-
Effect.timeout("10 seconds"),
102-
)
103-
104124
expect(yield* registered).toBe(false)
105125
yield* imageModel("test", "painter")
106126
expect(yield* until(true)).toBe(true)
@@ -110,6 +130,49 @@ describe("image_generate", () => {
110130
}),
111131
)
112132

133+
it.live("is not registered while config turns it off", () =>
134+
Effect.gen(function* () {
135+
yield* imageModel("test", "painter")
136+
yield* configure(false)
137+
yield* setup()
138+
expect(yield* registered).toBe(false)
139+
yield* configure({})
140+
expect(yield* until(true)).toBe(true)
141+
yield* configure(false)
142+
expect(yield* until(false)).toBe(false)
143+
}),
144+
)
145+
146+
it.live("uses the configured model unless the agent picks one, and falls back with a note", () =>
147+
Effect.gen(function* () {
148+
yield* imageModel("test", "painter", (model) => {
149+
model.capabilities.input = ["text", "image"]
150+
})
151+
yield* imageModel("other", "text-only", (model) => {
152+
model.capabilities.input = ["text"]
153+
})
154+
yield* configure(configuredModel("other", "text-only"))
155+
const { run, directory } = yield* setup()
156+
yield* Effect.promise(() => fs.writeFile(path.join(directory, "in.png"), Buffer.from(PNG, "base64")))
157+
158+
// The configured model beats the session's provider, and an explicit model beats both.
159+
const configured = yield* run({ prompt: "x" })
160+
expect(configured.output).toMatchObject({ model: "other/text-only" })
161+
expect(text(configured)).not.toContain("used")
162+
expect((yield* run({ prompt: "x", model: "test/painter" })).output).toMatchObject({ model: "test/painter" })
163+
164+
// A configured model that can't serve the call is skipped, and the result says why.
165+
const edit = yield* run({ prompt: "x", images: ["in.png"] })
166+
expect(edit.output).toMatchObject({ model: "test/painter" })
167+
expect(text(edit)).toContain("other/text-only doesn't accept input images; used test/painter.")
168+
169+
yield* configure(configuredModel("gone", "model"))
170+
const missing = yield* run({ prompt: "x" })
171+
expect(missing.output).toMatchObject({ model: "test/painter" })
172+
expect(text(missing)).toContain("gone/model isn't available; used test/painter.")
173+
}),
174+
)
175+
113176
it.live("reads input images, asks permission, and saves the result without overwriting", () =>
114177
Effect.gen(function* () {
115178
yield* imageModel("test", "painter", (model) => {

‎packages/protocol/openapi.json‎

Lines changed: 41 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/schema/src/config/experimental.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ export * as ConfigExperimental from "./experimental.js"
22

33
import { Schema } from "effect"
44
import { NonNegativeInt, optional } from "../schema.js"
5+
import { ConfigModel } from "./model.js"
56
import { ConfigPolicy } from "./policy.js"
67

78
export class Info extends Schema.Class<Info>("ConfigExperimental.Info")({
@@ -19,6 +20,15 @@ export class Info extends Schema.Class<Info>("ConfigExperimental.Info")({
1920
description:
2021
"Store new session media once in a deduplicated blob table instead of inline in every message row. Defaults to false.",
2122
}),
23+
image_generation: Schema.Union([
24+
Schema.Literal(false),
25+
Schema.Struct({ model: ConfigModel.Selection.pipe(optional) }),
26+
])
27+
.pipe(optional)
28+
.annotate({
29+
description:
30+
"Image generation tool settings. `model` sets the default image model (providerID/modelID) used when the agent doesn't choose one; set to false to disable the image_generate tool.",
31+
}),
2232
schedules: Schema.Boolean.pipe(optional).annotate({
2333
description: "Enable experimental scheduled tasks that wake a session with a prompt. Defaults to false.",
2434
}),

0 commit comments

Comments
 (0)