Let generate_image work from reference images, with an image editing model

generate_image takes reference images: files the run reaches as
[file:ID] refs, or the user's uploads by media id, up to four, each
checked like a picture source and limited to raster images.

Calls with references go to a new Image editing harness (no automatic
picks). While it has no model or is off, image generation takes them;
with its fallback on, image generation steps in after its pick fails.
Only models that take image input are tried with references, so none
drops them unseen. OpenRouter gets them on /images as input_references;
other providers as a multipart upload to /images/edits.

The Models pages show image editing's unset state as "Same as image
generation", and the models cache version is bumped so it shows at once.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-28 18:43:10 +01:00
co-authored by Claude Opus 5.5
parent 243c19e0c5
commit de28d4d9f7
22 changed files with 564 additions and 47 deletions
+54 -5
View File
@@ -1,9 +1,12 @@
import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
import HarnessError from "../harness-error";
import grabHarnessAdapter from "../providers/grab-harness-adapter";
import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
import withCandidates from "../resolve/with-candidates";
import type { HarnessImageOutput, HarnessUsage } from "../types";
import buildUsageCall from "../usage/build-usage-call";
import recordHarnessUsage from "../usage/record-harness-usage";
import orderImageCandidates from "./order-image-candidates";
type Params = {
prompt: string;
@@ -25,7 +28,9 @@ type Params = {
};
/**
* Makes images with the image generation harness. Returns their bytes.
* Makes images with the image generation harness, or with image editing
* when there are reference images (`orderImageCandidates` decides who
* tries). Returns their bytes.
*/
export default async function generateImage({
prompt,
@@ -36,15 +41,36 @@ export default async function generateImage({
bot_id,
workspace_id,
}: Params) {
const candidates = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id });
const has_references = Boolean(references?.length);
const edit = has_references ? await grabEdit({ bot_id, workspace_id }) : undefined;
/**
* Image generation turned off still leaves an image editing pick to
* work with.
*/
const image = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id }).catch(
(error: unknown) => {
if (edit?.candidates.length) return [];
throw error;
},
);
const ordered = orderImageCandidates({ references: has_references, image, edit });
if (has_references && !ordered.length && image.length) {
throw new HarnessError({
message: `None of the image models here takes reference images. A super admin can pick one under Image editing in Settings, under Models.`,
code: "unavailable",
});
}
const harness_of = new Map(ordered.map(({ candidate, harness }) => [candidate.key, harness]));
const { result } = await withCandidates<{ images: HarnessImageOutput[]; usage?: HarnessUsage }>({
harness: "image",
candidates,
harness: edit?.picked ? "image_edit" : "image",
candidates: ordered.map(({ candidate }) => candidate),
on_success: async ({ candidate, result: made, failed }): Promise<void> =>
await recordHarnessUsage({
call: buildUsageCall({
harness: "image",
harness: harness_of.get(candidate.key) || "image",
candidate,
failed,
usage: made.usage,
@@ -65,3 +91,26 @@ export default async function generateImage({
return result.images;
}
/**
* Image editing's setting and candidates, or undefined while it's off.
*/
async function grabEdit({
bot_id,
workspace_id,
}: {
bot_id?: number | string;
workspace_id?: number | string | null;
}) {
const setting = await grabHarnessSetting({ harness: "image_edit", bot_id, workspace_id });
if (!setting.enabled) return undefined;
return {
picked: Boolean(setting.model_id),
fallback: setting.fallback,
candidates: setting.model_id
? await resolveHarnessCandidates({ harness: "image_edit", bot_id, workspace_id })
: [],
};
}