tbenandClaude Opus 5.5
de28d4d9f7
generate_image takes reference images: files the run reaches as [file:ID] refs, or the user's uploads by media id, up to four, each checked like a picture source and limited to raster images. Calls with references go to a new Image editing harness (no automatic picks). While it has no model or is off, image generation takes them; with its fallback on, image generation steps in after its pick fails. Only models that take image input are tried with references, so none drops them unseen. OpenRouter gets them on /images as input_references; other providers as a multipart upload to /images/edits. The Models pages show image editing's unset state as "Same as image generation", and the models cache version is bumped so it shows at once. Co-Authored-By: Claude Opus 5.5 <[email protected]>
83 lines
2.8 KiB
TypeScript
83 lines
2.8 KiB
TypeScript
import { AppData } from "@/src/data/app-data";
|
|
import grabPictureSource from "@/src/functions/backend/db/profile-images/grab-picture-source";
|
|
import AppError from "@/src/utils/app-error";
|
|
import mediaFilePath from "@/src/utils/media-file-path";
|
|
import type { StorageScope } from "@/src/utils/storage-access";
|
|
|
|
/**
|
|
* Image types every image model takes as a reference.
|
|
*/
|
|
const ReferenceImageTypes = ["png", "jpeg", "jpg", "webp", "gif"];
|
|
|
|
type Params = {
|
|
user_id: number | string;
|
|
/**
|
|
* The files the run reaches, so a reference is only ever a file the
|
|
* bot could already read.
|
|
*/
|
|
scopes: StorageScope[];
|
|
/**
|
|
* Files by `[file:ID]` id.
|
|
*/
|
|
file_ids?: number[];
|
|
/**
|
|
* The user's own uploads, by media id.
|
|
*/
|
|
media_ids?: number[];
|
|
};
|
|
|
|
/**
|
|
* The reference images an image call passes on, as `data:` URLs. Each is
|
|
* checked the way a picture is (`grabPictureSource`: a file in the run's
|
|
* directories or the user's own upload), then to be a raster image within
|
|
* `AppData.ImageReferenceMaxBytes`. Throws on the first one that isn't.
|
|
*/
|
|
export default async function loadReferenceImages({
|
|
user_id,
|
|
scopes,
|
|
file_ids = [],
|
|
media_ids = [],
|
|
}: Params): Promise<string[]> {
|
|
const sources: { file_id?: number; media_id?: number }[] = [
|
|
...file_ids.map((file_id) => ({ file_id })),
|
|
...media_ids.map((media_id) => ({ media_id })),
|
|
];
|
|
|
|
if (sources.length > AppData.ImageReferenceMaxCount) {
|
|
throw new AppError(`Pass at most ${AppData.ImageReferenceMaxCount} reference images.`);
|
|
}
|
|
|
|
const urls: string[] = [];
|
|
|
|
for (const source of sources) {
|
|
const media = await grabPictureSource({ user_id, scopes, ...source });
|
|
const label = source.file_id ? `[file:${source.file_id}]` : `media id ${source.media_id}`;
|
|
const type = (media?.mime_type || "").replace(/^image\//, "").toLowerCase();
|
|
|
|
if (!media || media.media_type != "image" || !ReferenceImageTypes.includes(type)) {
|
|
throw new AppError(
|
|
`${label} can't be a reference: use a PNG, JPEG, WebP or GIF image.`,
|
|
);
|
|
}
|
|
|
|
const absolute = mediaFilePath({ write_path: media.media_write_path });
|
|
const disk = absolute ? Bun.file(absolute) : null;
|
|
|
|
if (!disk || !(await disk.exists())) {
|
|
throw new AppError(`${label} is missing on the server.`);
|
|
}
|
|
|
|
if (disk.size > AppData.ImageReferenceMaxBytes) {
|
|
throw new AppError(
|
|
`${label} is too large to pass on (at most ${Math.round(AppData.ImageReferenceMaxBytes / (1024 * 1024))} MB).`,
|
|
);
|
|
}
|
|
|
|
const base64 = Buffer.from(await disk.arrayBuffer()).toString("base64");
|
|
|
|
urls.push(`data:image/${type == "jpg" ? "jpeg" : type};base64,${base64}`);
|
|
}
|
|
|
|
return urls;
|
|
}
|