Let generate_image work from reference images, with an image editing model

generate_image takes reference images: files the run reaches as
[file:ID] refs, or the user's uploads by media id, up to four, each
checked like a picture source and limited to raster images.

Calls with references go to a new Image editing harness (no automatic
picks). While it has no model or is off, image generation takes them;
with its fallback on, image generation steps in after its pick fails.
Only models that take image input are tried with references, so none
drops them unseen. OpenRouter gets them on /images as input_references;
other providers as a multipart upload to /images/edits.

The Models pages show image editing's unset state as "Same as image
generation", and the models cache version is bumped so it shows at once.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/harness/image/generate-image.ts

@@ -1,9 +1,12 @@
+import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
+import HarnessError from "../harness-error";
 import grabHarnessAdapter from "../providers/grab-harness-adapter";
 import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
 import withCandidates from "../resolve/with-candidates";
 import type { HarnessImageOutput, HarnessUsage } from "../types";
 import buildUsageCall from "../usage/build-usage-call";
 import recordHarnessUsage from "../usage/record-harness-usage";
+import orderImageCandidates from "./order-image-candidates";
 
 type Params = {
     prompt: string;
@@ -25,7 +28,9 @@ type Params = {
 };
 
 /**
- * Makes images with the image generation harness. Returns their bytes.
+ * Makes images with the image generation harness, or with image editing
+ * when there are reference images (`orderImageCandidates` decides who
+ * tries). Returns their bytes.
  */
 export default async function generateImage({
     prompt,
@@ -36,15 +41,36 @@ export default async function generateImage({
     bot_id,
     workspace_id,
 }: Params) {
-    const candidates = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id });
+    const has_references = Boolean(references?.length);
+    const edit = has_references ? await grabEdit({ bot_id, workspace_id }) : undefined;
+    /**
+     * Image generation turned off still leaves an image editing pick to
+     * work with.
+     */
+    const image = await resolveHarnessCandidates({ harness: "image", bot_id, workspace_id }).catch(
+        (error: unknown) => {
+            if (edit?.candidates.length) return [];
+            throw error;
+        },
+    );
+    const ordered = orderImageCandidates({ references: has_references, image, edit });
+
+    if (has_references && !ordered.length && image.length) {
+        throw new HarnessError({
+            message: `None of the image models here takes reference images. A super admin can pick one under Image editing in Settings, under Models.`,
+            code: "unavailable",
+        });
+    }
+
+    const harness_of = new Map(ordered.map(({ candidate, harness }) => [candidate.key, harness]));
 
     const { result } = await withCandidates<{ images: HarnessImageOutput[]; usage?: HarnessUsage }>({
-        harness: "image",
-        candidates,
+        harness: edit?.picked ? "image_edit" : "image",
+        candidates: ordered.map(({ candidate }) => candidate),
         on_success: async ({ candidate, result: made, failed }): Promise<void> =>
             await recordHarnessUsage({
                 call: buildUsageCall({
-                    harness: "image",
+                    harness: harness_of.get(candidate.key) || "image",
                     candidate,
                     failed,
                     usage: made.usage,
@@ -65,3 +91,26 @@ export default async function generateImage({
 
     return result.images;
 }
+
+/**
+ * Image editing's setting and candidates, or undefined while it's off.
+ */
+async function grabEdit({
+    bot_id,
+    workspace_id,
+}: {
+    bot_id?: number | string;
+    workspace_id?: number | string | null;
+}) {
+    const setting = await grabHarnessSetting({ harness: "image_edit", bot_id, workspace_id });
+
+    if (!setting.enabled) return undefined;
+
+    return {
+        picked: Boolean(setting.model_id),
+        fallback: setting.fallback,
+        candidates: setting.model_id
+            ? await resolveHarnessCandidates({ harness: "image_edit", bot_id, workspace_id })
+            : [],
+    };
+}

src/harness/image/load-reference-images.ts

@@ -0,0 +1,82 @@
+import { AppData } from "@/src/data/app-data";
+import grabPictureSource from "@/src/functions/backend/db/profile-images/grab-picture-source";
+import AppError from "@/src/utils/app-error";
+import mediaFilePath from "@/src/utils/media-file-path";
+import type { StorageScope } from "@/src/utils/storage-access";
+
+/**
+ * Image types every image model takes as a reference.
+ */
+const ReferenceImageTypes = ["png", "jpeg", "jpg", "webp", "gif"];
+
+type Params = {
+    user_id: number | string;
+    /**
+     * The files the run reaches, so a reference is only ever a file the
+     * bot could already read.
+     */
+    scopes: StorageScope[];
+    /**
+     * Files by `[file:ID]` id.
+     */
+    file_ids?: number[];
+    /**
+     * The user's own uploads, by media id.
+     */
+    media_ids?: number[];
+};
+
+/**
+ * The reference images an image call passes on, as `data:` URLs. Each is
+ * checked the way a picture is (`grabPictureSource`: a file in the run's
+ * directories or the user's own upload), then to be a raster image within
+ * `AppData.ImageReferenceMaxBytes`. Throws on the first one that isn't.
+ */
+export default async function loadReferenceImages({
+    user_id,
+    scopes,
+    file_ids = [],
+    media_ids = [],
+}: Params): Promise<string[]> {
+    const sources: { file_id?: number; media_id?: number }[] = [
+        ...file_ids.map((file_id) => ({ file_id })),
+        ...media_ids.map((media_id) => ({ media_id })),
+    ];
+
+    if (sources.length > AppData.ImageReferenceMaxCount) {
+        throw new AppError(`Pass at most ${AppData.ImageReferenceMaxCount} reference images.`);
+    }
+
+    const urls: string[] = [];
+
+    for (const source of sources) {
+        const media = await grabPictureSource({ user_id, scopes, ...source });
+        const label = source.file_id ? `[file:${source.file_id}]` : `media id ${source.media_id}`;
+        const type = (media?.mime_type || "").replace(/^image\//, "").toLowerCase();
+
+        if (!media || media.media_type != "image" || !ReferenceImageTypes.includes(type)) {
+            throw new AppError(
+                `${label} can't be a reference: use a PNG, JPEG, WebP or GIF image.`,
+            );
+        }
+
+        const absolute = mediaFilePath({ write_path: media.media_write_path });
+        const disk = absolute ? Bun.file(absolute) : null;
+
+        if (!disk || !(await disk.exists())) {
+            throw new AppError(`${label} is missing on the server.`);
+        }
+
+        if (disk.size > AppData.ImageReferenceMaxBytes) {
+            throw new AppError(
+                `${label} is too large to pass on (at most ${Math.round(AppData.ImageReferenceMaxBytes / (1024 * 1024))} MB).`,
+            );
+        }
+
+        const base64 = Buffer.from(await disk.arrayBuffer()).toString("base64");
+
+        urls.push(`data:image/${type == "jpg" ? "jpeg" : type};base64,${base64}`);
+    }
+
+    return urls;
+}

src/harness/image/order-image-candidates.test.ts

@@ -0,0 +1,105 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessCandidate } from "../types";
+import orderImageCandidates from "./order-image-candidates";
+
+function candidate({ slug, takes_images }: { slug: string; takes_images: boolean }): HarnessCandidate {
+    return {
+        key: `1:${slug}`,
+        provider: { id: 1, kind: "openrouter", base_url: "", name: "OpenRouter" } as HarnessCandidate["provider"],
+        model: {
+            slug,
+            input_modalities: takes_images ? ["text", "image"] : ["text"],
+            output_modalities: ["image"],
+            supports_tools: false,
+        },
+        source: "pinned",
+    };
+}
+
+const flux = candidate({ slug: "flux", takes_images: true });
+const recraft = candidate({ slug: "recraft", takes_images: false });
+const qwen_edit = candidate({ slug: "qwen-image-edit", takes_images: true });
+
+function slugs(list: ReturnType<typeof orderImageCandidates>) {
+    return list.map(({ candidate, harness }) => `${harness}:${candidate.model.slug}`);
+}
+
+describe("orderImageCandidates", () => {
+    test("without references, image generation's models as they are", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: false,
+                    image: [recraft, flux],
+                    edit: { picked: true, fallback: true, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image:recraft", "image:flux"]);
+    });
+
+    test("with references and no edit pick, only image models that take images", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [recraft, flux],
+                    edit: { picked: false, fallback: true, candidates: [] },
+                }),
+            ),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("image editing off goes to image generation", () => {
+        expect(
+            slugs(orderImageCandidates({ references: true, image: [recraft, flux] })),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("the edit pick first, then image generation with fallback on", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux, recraft],
+                    edit: { picked: true, fallback: true, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image_edit:qwen-image-edit", "image:flux"]);
+    });
+
+    test("fallback off keeps to the edit pick", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: false, candidates: [qwen_edit] },
+                }),
+            ),
+        ).toEqual(["image_edit:qwen-image-edit"]);
+    });
+
+    test("an edit pick that can't run falls to image generation", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: false, candidates: [] },
+                }),
+            ),
+        ).toEqual(["image:flux"]);
+    });
+
+    test("a model in both lists is tried once, as the edit pick", () => {
+        expect(
+            slugs(
+                orderImageCandidates({
+                    references: true,
+                    image: [flux],
+                    edit: { picked: true, fallback: true, candidates: [flux] },
+                }),
+            ),
+        ).toEqual(["image_edit:flux"]);
+    });
+});

src/harness/image/order-image-candidates.ts

@@ -0,0 +1,65 @@
+import type { HarnessCandidate } from "../types";
+
+export type ImageCandidate = {
+    candidate: HarnessCandidate;
+    /**
+     * The harness it was picked for, for its usage row.
+     */
+    harness: "image" | "image_edit";
+};
+
+type Params = {
+    /**
+     * Whether the call passes reference images.
+     */
+    references: boolean;
+    /**
+     * Image generation's candidates.
+     */
+    image: HarnessCandidate[];
+    /**
+     * Image editing's setting and candidates; undefined when it's off.
+     */
+    edit?: {
+        /**
+         * Whether a model is picked for it.
+         */
+        picked: boolean;
+        fallback: boolean;
+        candidates: HarnessCandidate[];
+    };
+};
+
+/**
+ * The models an image call tries, in order. Without references: image
+ * generation's. With them: image editing's pick, then (with fallback on)
+ * the image generation models that take images; only those while image
+ * editing has no usable model or is off. A model that can't take an image
+ * is never tried with references, since it would drop them unseen.
+ */
+export default function orderImageCandidates({ references, image, edit }: Params): ImageCandidate[] {
+    if (!references) {
+        return image.map((candidate) => ({ candidate, harness: "image" }));
+    }
+
+    const takes_images = image
+        .filter((candidate) => candidate.model.input_modalities.includes("image"))
+        .map((candidate) => ({ candidate, harness: "image" as const }));
+
+    const edits = edit?.picked
+        ? edit.candidates.map((candidate) => ({ candidate, harness: "image_edit" as const }))
+        : [];
+
+    const ordered = !edits.length
+        ? takes_images
+        : [...edits, ...(edit?.fallback ? takes_images : [])];
+
+    const seen = new Set<string>();
+
+    return ordered.filter(({ candidate }) => {
+        if (seen.has(candidate.key)) return false;
+        seen.add(candidate.key);
+
+        return true;
+    });
+}