Let generate_image work from reference images, with an image editing model

generate_image takes reference images: files the run reaches as
[file:ID] refs, or the user's uploads by media id, up to four, each
checked like a picture source and limited to raster images.

Calls with references go to a new Image editing harness (no automatic
picks). While it has no model or is off, image generation takes them;
with its fallback on, image generation steps in after its pick fails.
Only models that take image input are tried with references, so none
drops them unseen. OpenRouter gets them on /images as input_references;
other providers as a multipart upload to /images/edits.

The Models pages show image editing's unset state as "Same as image
generation", and the models cache version is bumped so it shows at once.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/harness/catalog/guess-model-modalities.ts

@@ -11,7 +11,7 @@ type Guess = Pick<
 
 const Transcription = /whisper|transcri|(?:^|[-_/])(stt|asr)(?=$|[-_:.])|scribe/i;
 const Speech = /(?:^|[-_/])tts(?=$|[-_:.])|speech|voice/i;
-const ImageOut = /dall-e|gpt-image|imagen|flux|stable-diffusion|sdxl|image-gen/i;
+const ImageOut = /dall-e|gpt-image|imagen|flux|stable-diffusion|sdxl|image-gen|qwen-image|kontext|seedream|image-edit/i;
 const Embedding = /embed/i;
 /**
  * Chat families that read images.

src/harness/catalog/model-serves-harness.test.ts

@@ -72,4 +72,19 @@ describe("modelServesHarness", () => {
             }),
         ).toBe(true);
     });
+
+    test("image editing takes and makes images", () => {
+        expect(
+            modelServesHarness({
+                model: { ...chat, output_modalities: ["image"] },
+                harness: "image_edit",
+            }),
+        ).toBe(false);
+        expect(
+            modelServesHarness({
+                model: { ...chat, input_modalities: ["text", "image"], output_modalities: ["image"] },
+                harness: "image_edit",
+            }),
+        ).toBe(true);
+    });
 });

src/harness/catalog/model-serves-harness.ts

@@ -17,6 +17,7 @@ type Params = {
  *   audio: free ones often drop it)
  * - speech: a text-to-speech model
  * - image: makes images
+ * - image_edit: makes images from reference images (takes and makes them)
  * - embedding: turns text into vectors (never a chat model)
  */
 export default function modelServesHarness({ model, harness }: Params) {
@@ -41,6 +42,9 @@ export default function modelServesHarness({ model, harness }: Params) {
         case "image":
             return output.has("image");
 
+        case "image_edit":
+            return input.has("image") && output.has("image");
+
         case "embedding":
             return output.has("embeddings");
     }

src/harness/image/generate-image.ts

@@ -1,9 +1,12 @@
+import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
+import HarnessError from "../harness-error";
 import grabHarnessAdapter from "../providers/grab-harness-adapter";
 import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
 import withCandidates from "../resolve/with-candidates";
 import type { HarnessImageOutput, HarnessUsage } from "../types";
 import buildUsageCall from "../usage/build-usage-call";
 import recordHarnessUsage from "../usage/record-harness-usage";
+import orderImageCandidates from "./order-image-candidates";
 
 type Params = {
     prompt: string;
@@ -25,7 +28,9 @@ type Params = {
 };
 
 /**
- * Makes images with the image generation harness. Returns their bytes.
+ * Makes images with the image generation harness, or with image editing
+ * when there are reference images (`orderImageCandidates` decides who
+ * tries). Returns their bytes.
  */
 export default async function generateImage({
     prompt,
@@ -36,15 +41,36 @@ export default async function generateImage({
     bot_id,
     workspace_id,
 }: Params) {
-    const candidates = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id });
+    const has_references = Boolean(references?.length);
+    const edit = has_references ? await grabEdit({ bot_id, workspace_id }) : undefined;
+    /**
+     * Image generation turned off still leaves an image editing pick to
+     * work with.
+     */
+    const image = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id }).catch(
+        (error: unknown) => {
+            if (edit?.candidates.length) return [];
+            throw error;
+        },
+    );
+    const ordered = orderImageCandidates({ references: has_references, image, edit });
+
+    if (has_references && !ordered.length && image.length) {
+        throw new HarnessError({
+            message: `None of the image models here takes reference images. A super admin can pick one under Image editing in Settings, under Models.`,
+            code: "unavailable",
+        });
+    }
+
+    const harness_of = new Map(ordered.map(({ candidate, harness }) => [candidate.key, harness]));
 
     const { result } = await withCandidates<{ images: HarnessImageOutput[]; usage?: HarnessUsage }>({
-        harness: "image",
-        candidates,
+        harness: edit?.picked ? "image_edit" : "image",
+        candidates: ordered.map(({ candidate }) => candidate),
         on_success: async ({ candidate, result: made, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
-                    harness: "image",
+                    harness: harness_of.get(candidate.key) || "image",
                     candidate,
                     failed,
                     usage: made.usage,
@@ -65,3 +91,26 @@ export default async function generateImage({
 
     return result.images;
 }
+
+/**
+ * Image editing's setting and candidates, or undefined while it's off.
+ */
+async function grabEdit({
+    bot_id,
+    workspace_id,
+}: {
+    bot_id?: number | string;
+    workspace_id?: number | string | null;
+}) {
+    const setting = await grabHarnessSetting({ harness: "image_edit", bot_id, workspace_id });
+
+    if (!setting.enabled) return undefined;
+
+    return {
+        picked: Boolean(setting.model_id),
+        fallback: setting.fallback,
+        candidates: setting.model_id
+            ? await resolveHarnessCandidates({ harness: "image_edit", bot_id, workspace_id })
+            : [],
+    };
+}

src/harness/image/load-reference-images.ts

@@ -0,0 +1,82 @@
+import { AppData } from "@/src/data/app-data";
+import grabPictureSource from "@/src/functions/backend/db/profile-images/grab-picture-source";
+import AppError from "@/src/utils/app-error";
+import mediaFilePath from "@/src/utils/media-file-path";
+import type { StorageScope } from "@/src/utils/storage-access";
+
+/**
+ * Image types every image model takes as a reference.
+ */
+const ReferenceImageTypes = ["png", "jpeg", "jpg", "webp", "gif"];
+
+type Params = {
+    user_id: number | string;
+    /**
+     * The files the run reaches, so a reference is only ever a file the
+     * bot could already read.
+     */
+    scopes: StorageScope[];
+    /**
+     * Files by `[file:ID]` id.
+     */
+    file_ids?: number[];
+    /**
+     * The user's own uploads, by media id.
+     */
+    media_ids?: number[];
+};
+
+/**
+ * The reference images an image call passes on, as `data:` URLs. Each is
+ * checked the way a picture is (`grabPictureSource`: a file in the run's
+ * directories or the user's own upload), then to be a raster image within
+ * `AppData.ImageReferenceMaxBytes`. Throws on the first one that isn't.
+ */
+export default async function loadReferenceImages({
+    user_id,
+    scopes,
+    file_ids = [],
+    media_ids = [],
+}: Params): Promise<string[]> {
+    const sources: { file_id?: number; media_id?: number }[] = [
+        ...file_ids.map((file_id) => ({ file_id })),
+        ...media_ids.map((media_id) => ({ media_id })),
+    ];
+
+    if (sources.length > AppData.ImageReferenceMaxCount) {
+        throw new AppError(`Pass at most ${AppData.ImageReferenceMaxCount} reference images.`);
+    }
+
+    const urls: string[] = [];
+
+    for (const source of sources) {
+        const media = await grabPictureSource({ user_id, scopes, ...source });
+        const label = source.file_id ? `[file:${source.file_id}]` : `media id ${source.media_id}`;
+        const type = (media?.mime_type || "").replace(/^image\//, "").toLowerCase();
+
+        if (!media || media.media_type != "image" || !ReferenceImageTypes.includes(type)) {
+            throw new AppError(
+                `${label} can't be a reference: use a PNG, JPEG, WebP or GIF image.`,
+            );
+        }
+
+        const absolute = mediaFilePath({ write_path: media.media_write_path });
+        const disk = absolute ? Bun.file(absolute) : null;
+
+        if (!disk || !(await disk.exists())) {
+            throw new AppError(`${label} is missing on the server.`);
+        }
+
+        if (disk.size > AppData.ImageReferenceMaxBytes) {
+            throw new AppError(
+                `${label} is too large to pass on (at most ${Math.round(AppData.ImageReferenceMaxBytes / (1024 * 1024))} MB).`,
+            );
+        }
+
+        const base64 = Buffer.from(await disk.arrayBuffer()).toString("base64");
+
+        urls.push(`data:image/${type == "jpg" ? "jpeg" : type};base64,${base64}`);
+    }
+
+    return urls;
+}

src/harness/image/order-image-candidates.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessCandidate } from "../types";
+import orderImageCandidates from "./order-image-candidates";
+
+function candidate({ slug, takes_images }: { slug: string; takes_images: boolean }): HarnessCandidate {
+    return {
+        key: `1:${slug}`,
+        provider: { id: 1, kind: "openrouter", base_url: "", name: "OpenRouter" } as HarnessCandidate["provider"],
+        model: {
+            slug,
+            input_modalities: takes_images ? ["text", "image"] : ["text"],
+            output_modalities: ["image"],
+            supports_tools: false,
+        },
+        source: "pinned",
+    };
+}
+
+const flux = candidate({ slug: "flux", takes_images: true });
+const recraft = candidate({ slug: "recraft", takes_images: false });
+const qwen_edit = candidate({ slug: "qwen-image-edit", takes_images: true });
+
+function slugs(list: ReturnType<typeof orderImageCandidates>) {
+    return list.map(({ candidate, harness }) => `${harness}:${candidate.model.slug}`);
+}
+
+describe("orderImageCandidates", () => {
+    test("without references, image generation's models as they are", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: false,
+                    image: [recraft, flux],
+                    edit: { picked: true, fallback: true, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image:recraft", "image:flux"]);
+    });
+
+    test("with references and no edit pick, only image models that take images", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [recraft, flux],
+                    edit: { picked: false, fallback: true, candidates: [] },
+                }),
+            ),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("image editing off goes to image generation", () => {
+        expect(
+            slugs(orderImageCandidates({ references: true, image: [recraft, flux] })),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("the edit pick first, then image generation with fallback on", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux, recraft],
+                    edit: { picked: true, fallback: true, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image_edit:qwen-image-edit", "image:flux"]);
+    });
+
+    test("fallback off keeps to the edit pick", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: false, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image_edit:qwen-image-edit"]);
+    });
+
+    test("an edit pick that can't run falls to image generation", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: false, candidates: [] },
+                }),
+            ),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("a model in both lists is tried once, as the edit pick", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: true, candidates: [flux] },
+                }),
+            ),
+        ).toEqual(["image_edit:flux"]);
+    });
+});

src/harness/image/order-image-candidates.ts

@@ -0,0 +1,65 @@
+import type { HarnessCandidate } from "../types";
+
+export type ImageCandidate = {
+    candidate: HarnessCandidate;
+    /**
+     * The harness it was picked for, for its usage row.
+     */
+    harness: "image" | "image_edit";
+};
+
+type Params = {
+    /**
+     * Whether the call passes reference images.
+     */
+    references: boolean;
+    /**
+     * Image generation's candidates.
+     */
+    image: HarnessCandidate[];
+    /**
+     * Image editing's setting and candidates; undefined when it's off.
+     */
+    edit?: {
+        /**
+         * Whether a model is picked for it.
+         */
+        picked: boolean;
+        fallback: boolean;
+        candidates: HarnessCandidate[];
+    };
+};
+
+/**
+ * The models an image call tries, in order. Without references: image
+ * generation's. With them: image editing's pick, then (with fallback on)
+ * the image generation models that take images; only those while image
+ * editing has no usable model or is off. A model that can't take an image
+ * is never tried with references, since it would drop them unseen.
+ */
+export default function orderImageCandidates({ references, image, edit }: Params): ImageCandidate[] {
+    if (!references) {
+        return image.map((candidate) => ({ candidate, harness: "image" }));
+    }
+
+    const takes_images = image
+        .filter((candidate) => candidate.model.input_modalities.includes("image"))
+        .map((candidate) => ({ candidate, harness: "image" as const }));
+
+    const edits = edit?.picked
+        ? edit.candidates.map((candidate) => ({ candidate, harness: "image_edit" as const }))
+        : [];
+
+    const ordered = !edits.length
+        ? takes_images
+        : [...edits, ...(edit?.fallback ? takes_images : [])];
+
+    const seen = new Set<string>();
+
+    return ordered.filter(({ candidate }) => {
+        if (seen.has(candidate.key)) return false;
+        seen.add(candidate.key);
+
+        return true;
+    });
+}

src/harness/providers/openai-compatible/generate-image.ts

@@ -1,4 +1,5 @@
 import { AppData } from "@/src/data/app-data";
+import bufferFromBase64 from "@/src/utils/buffer-from-base-64";
 import HarnessError from "../../harness-error";
 import type { HarnessAdapter, HarnessImageOutput } from "../../types";
 import grabProviderPreset from "../grab-provider-preset";
@@ -6,8 +7,10 @@ import providerRequest from "./provider-request";
 
 /**
  * Image generation at the provider's `image_path` (`/images` on
- * OpenRouter, `/images/generations` elsewhere). Images come back as
- * base64, or as URLs that are downloaded here.
+ * OpenRouter, `/images/generations` elsewhere). With reference images,
+ * OpenRouter takes them on the same call (`input_references`); the others
+ * get them as a multipart upload at `image_edit_path` (`/images/edits`).
+ * Images come back as base64, or as URLs that are downloaded here.
  */
 const generateImage: HarnessAdapter["generateImage"] = async ({
     candidate,
@@ -20,29 +23,45 @@ const generateImage: HarnessAdapter["generateImage"] = async ({
     const { provider, model } = candidate;
     const preset = grabProviderPreset({ kind: provider.kind });
     const is_openrouter = provider.kind == "openrouter";
+    const edits = Boolean(references?.length) && !is_openrouter;
 
-    const res = await providerRequest({
-        provider,
-        path: preset.image_path,
-        json: {
-            model: model.slug,
-            prompt,
-            ...(n ? { n } : {}),
-            ...(size ? { size } : {}),
-            ...(is_openrouter
-                ? references?.length
-                    ? {
-                          input_references: references.map((url) => ({
-                              type: "image_url",
-                              image_url: { url },
-                          })),
-                      }
-                    : {}
-                : { response_format: "b64_json" }),
-        },
-        signal,
-        candidate_key: candidate.key,
-    });
+    const res = edits
+        ? await providerRequest({
+              provider,
+              path: preset.image_edit_path,
+              form: await editForm({
+                  model: model.slug,
+                  prompt,
+                  n,
+                  size,
+                  references: references || [],
+                  signal,
+              }),
+              signal,
+              candidate_key: candidate.key,
+          })
+        : await providerRequest({
+              provider,
+              path: preset.image_path,
+              json: {
+                  model: model.slug,
+                  prompt,
+                  ...(n ? { n } : {}),
+                  ...(size ? { size } : {}),
+                  ...(is_openrouter
+                      ? references?.length
+                          ? {
+                                input_references: references.map((url) => ({
+                                    type: "image_url",
+                                    image_url: { url },
+                                })),
+                            }
+                          : {}
+                      : { response_format: "b64_json" }),
+              },
+              signal,
+              candidate_key: candidate.key,
+          });
 
     const json = await res.json().catch(() => null);
     const items: any[] = Array.isArray(json?.data) ? json.data : [];
@@ -98,3 +117,59 @@ const generateImage: HarnessAdapter["generateImage"] = async ({
 };
 
 export default generateImage;
+
+type EditFormParams = {
+    model: string;
+    prompt: string;
+    n?: number;
+    size?: string;
+    references: string[];
+    signal: AbortSignal;
+};
+
+/**
+ * The `/images/edits` upload: the prompt and every reference image as a
+ * file (`image[]` when there are several, as gpt-image takes them).
+ */
+async function editForm({ model, prompt, n, size, references, signal }: EditFormParams) {
+    const form = new FormData();
+
+    form.append("model", model);
+    form.append("prompt", prompt);
+    if (n) form.append("n", String(n));
+    if (size) form.append("size", size);
+
+    const field = references.length > 1 ? "image[]" : "image";
+
+    for (let index = 0; index < references.length; index++) {
+        const { blob, ext } = await referenceBlob({ url: references[index]!, signal });
+
+        form.append(field, blob, `reference-${index + 1}.${ext}`);
+    }
+
+    return form;
+}
+
+/**
+ * A reference's bytes: decoded from a `data:` URL, downloaded otherwise.
+ */
+async function referenceBlob({ url, signal }: { url: string; signal: AbortSignal }) {
+    if (url.startsWith("data:")) {
+        const { buffer, ext } = await bufferFromBase64(url);
+        const type = url.slice(5, url.indexOf(";")) || "image/png";
+
+        return { blob: new Blob([new Uint8Array(buffer)], { type }), ext };
+    }
+
+    const download = await fetch(url, {
+        signal: AbortSignal.any([signal, AbortSignal.timeout(AppData.HarnessRequestTimeoutMs)]),
+    });
+
+    if (!download.ok) {
+        throw new HarnessError({ message: `A reference image couldn't be downloaded.` });
+    }
+
+    const type = download.headers.get("content-type")?.split(";")[0] || "image/png";
+
+    return { blob: new Blob([await download.arrayBuffer()], { type }), ext: type.split("/")[1] || "png" };
+}

src/harness/tools/storage/generate-image-tool.ts

@@ -1,6 +1,9 @@
 import { z } from "zod";
 import recordBotFile from "@/src/functions/backend/db/bot-files/record-bot-file";
+import { AppData } from "@/src/data/app-data";
 import generateImage from "@/src/harness/image/generate-image";
+import loadReferenceImages from "@/src/harness/image/load-reference-images";
+import parseFileRef from "@/src/utils/parse-file-ref";
 import type { HarnessTool } from "../../types";
 import {
     describeScope,
@@ -12,6 +15,8 @@ import {
 
 type Args = {
     prompt: string;
+    references?: (string | number)[];
+    reference_media_ids?: number[];
     path?: string;
     size?: string;
     attach?: boolean;
@@ -19,18 +24,30 @@ type Args = {
 };
 
 /**
- * Makes an image with the image generation harness and saves it in the
- * bot's files, so it can be sent, shown again later or used by another
- * run. The bytes stay in the app's storage, never on a computer.
+ * Makes an image with the image generation harness (image editing when it
+ * passes reference images: files the run reaches or the user's uploads)
+ * and saves it in the bot's files, so it can be sent, shown again later or
+ * used by another run. The bytes stay in the app's storage, never on a
+ * computer.
  */
 export default function generateImageTool(context: StorageToolContext): HarnessTool<Args> {
     const own = context.scopes[0];
 
     return {
         name: "generate_image",
-        description: `Make an image from a description and save it in your files. Describe what should be in it, in one clear prompt. The image is saved (so you can send it now and show it later) and can ride on this reply with "attach".`,
+        description: `Make an image from a description and save it in your files. Describe what should be in it, in one clear prompt. To work from existing images (a logo to rework, a style or palette to follow, a photo to change), pass them as "references" ([file:ID] refs from list_files) or "reference_media_ids" (images the user uploaded), up to ${AppData.ImageReferenceMaxCount}, and say in the prompt what to take from each. The image is saved (so you can send it now and show it later) and can ride on this reply with "attach".`,
         schema: z.object({
             prompt: z.string().min(3).max(4000),
+            references: z
+                .array(z.union([z.string().max(60), z.number().int().positive()]))
+                .max(AppData.ImageReferenceMaxCount)
+                .optional()
+                .describe(`Images in your files to work from, by [file:ID] ref, e.g. ["[file:12]"]`),
+            reference_media_ids: z
+                .array(z.number().int().positive())
+                .max(AppData.ImageReferenceMaxCount)
+                .optional()
+                .describe(`Images the user uploaded to work from, by media id`),
             path: z
                 .string()
                 .max(300)
@@ -55,8 +72,22 @@ export default function generateImageTool(context: StorageToolContext): HarnessT
                     .join(", ")}.`;
             }
 
+            const file_ids = (args.references || []).map((ref) => parseFileRef({ ref }));
+
+            if (file_ids.some((id) => !id)) {
+                return `Give each reference as its [file:ID] ref, e.g. "[file:12]". list_files lists them.`;
+            }
+
+            const references = await loadReferenceImages({
+                user_id: context.user_id,
+                scopes: context.scopes,
+                file_ids: file_ids as number[],
+                media_ids: args.reference_media_ids,
+            });
+
             const images = await generateImage({
                 prompt: args.prompt,
+                references,
                 size: args.size,
                 signal,
                 bot_id: context.bot_id,

src/harness/types.ts

@@ -157,6 +157,7 @@ export type HarnessUsageKind =
     | "transcription"
     | "speech"
     | "image"
+    | "image_edit"
     | "embedding";
 
 export type HarnessChatParams = {

src/harness/usage/types.ts

@@ -58,7 +58,7 @@ export type HarnessCallSettings = {
 };
 
 export type HarnessUsageCall = {
-    harness: "text" | "vision" | "transcription" | "speech" | "image" | "embedding";
+    harness: "text" | "vision" | "transcription" | "speech" | "image" | "image_edit" | "embedding";
     provider_id?: number;
     provider_name?: string;
     model_id?: number;