Let generate_image work from reference images, with an image editing model

generate_image takes reference images: files the run reaches as
[file:ID] refs, or the user's uploads by media id, up to four, each
checked like a picture source and limited to raster images.

Calls with references go to a new Image editing harness (no automatic
picks). While it has no model or is off, image generation takes them;
with its fallback on, image generation steps in after its pick fails.
Only models that take image input are tried with references, so none
drops them unseen. OpenRouter gets them on /images as input_references;
other providers as a multipart upload to /images/edits.

The Models pages show image editing's unset state as "Same as image
generation", and the models cache version is bumped so it shows at once.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-28 18:43:10 +01:00
co-authored by Claude Opus 5.5
parent 243c19e0c5
commit de28d4d9f7
22 changed files with 564 additions and 47 deletions
@@ -1,4 +1,5 @@
import Span from "@moduletrace/twui/layout/Span";
import { harnessFallbackTitle } from "@/src/dict/harness-dict";
import type { HarnessOverview } from "@/src/types";
type Props = {
@@ -9,8 +10,11 @@ type Props = {
* What the harness runs on right now, as saved, in one line.
*/
export default function HarnessCurrentModel({ harness }: Props) {
const falls_back_title = harnessFallbackTitle({ harness: harness.harness });
const text = !harness.enabled
? "Turned off."
? falls_back_title
? `Turned off: ${falls_back_title.toLowerCase()} does it.`
: "Turned off."
: harness.current
? `Now using ${harness.current.name} on ${harness.current.provider_name}${
harness.current.source == "auto" ? ", picked automatically" : ""
@@ -21,7 +25,9 @@ export default function HarnessCurrentModel({ harness }: Props) {
}`
: harness.required
? "Nothing to run on yet: no provider has a usable key."
: !harness.automatic
: falls_back_title && !harness.model_id
? `${falls_back_title} does it until a model is picked.`
: !harness.automatic
? "Not available until a model is picked."
: "No free model can do this right now. Pick one to turn it on.";
@@ -13,6 +13,7 @@ type Props = {
model_id: number | null;
provider_name?: string;
automatic: boolean;
falls_back_title?: string;
onChange: (model_id: number | null) => void;
};
@@ -26,6 +27,7 @@ export default function HarnessModelSelect({
model_id,
provider_name,
automatic,
falls_back_title,
onChange,
}: Props) {
return (
@@ -35,7 +37,13 @@ export default function HarnessModelSelect({
label="Model"
showLabel
placeholder="Search models"
options={harnessModelSelectOptions({ models, model_id, provider_name, automatic })}
options={harnessModelSelectOptions({
models,
model_id,
provider_name,
automatic,
falls_back_title,
})}
defaultValues={[model_id ? String(model_id) : AutoModelValue]}
changeHandler={(value) => {
const picked = Array.isArray(value) ? value[0] : value;
@@ -1,4 +1,5 @@
import Checkbox from "@moduletrace/twui/form/Checkbox";
import { harnessFallbackTitle } from "@/src/dict/harness-dict";
import { HarnessParameterHarnesses } from "@/src/dict/harness-parameter-dict";
import type { HarnessFormValue, HarnessOverview } from "@/src/types";
import SettingsGroup from "../../(partials)/settings-group";
@@ -40,11 +41,14 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
harness.harness as (typeof HarnessParameterHarnesses)[number],
);
const on_bot_page = harness.default !== undefined;
const falls_back_title = harnessFallbackTitle({ harness: harness.harness });
const default_name = harness.default?.enabled === false
? "turned off"
: harness.default?.current
? `${harness.default.current.name} on ${harness.default.current.provider_name}`
: "not available";
: falls_back_title
? `same as ${falls_back_title.toLowerCase()}`
: "not available";
return (
<SettingsGroup title={harness.title} description={harness.description}>
@@ -74,6 +78,7 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
model_id={value.model_id}
provider_name={provider?.name}
automatic={harness.automatic}
falls_back_title={falls_back_title}
onChange={(model_id) => onChange({ ...value, model_id, voice: undefined })}
/>
)}
@@ -90,6 +95,14 @@ export default function HarnessSettingGroup({ harness, value, onChange }: Props)
changeHandler={(fallback) => onChange({ ...value, fallback })}
/>
) : null}
{!value.inherit && value.model_id && falls_back_title ? (
<Checkbox
title={`Fall back to ${falls_back_title.toLowerCase()} if this model fails`}
label={`Fall back to ${falls_back_title.toLowerCase()} if this model fails`}
checked={value.fallback}
changeHandler={(fallback) => onChange({ ...value, fallback })}
/>
) : null}
{shows_params && params_model ? (
<HarnessParameterModal
id={id}
@@ -18,26 +18,35 @@ type Params = {
* available".
*/
automatic: boolean;
/**
* The harness that does the work while none is picked (image
* generation, for image editing): the empty choice names it.
*/
falls_back_title?: string;
};
/**
* The model picker's options: automatic (the best free model, on the
* picked provider if there is one), then every model that can do the job
* (free ones first, as the server sorts them). A harness without
* automatic picks starts with "Not available" instead.
* automatic picks starts with "Not available" instead, or with the
* harness it falls back to.
*/
export default function harnessModelSelectOptions({
models,
model_id,
provider_name,
automatic,
falls_back_title,
}: Params): TWUISelectOptionObject[] {
const has_free = models.some((model) => model.free);
return [
{
value: AutoModelValue,
title: !automatic
title: falls_back_title
? `Same as ${falls_back_title.toLowerCase()} (pick a model to use its own)`
: !automatic
? "Not available (pick a model to turn it on)"
: provider_name
? has_free
@@ -8,5 +8,6 @@ export const UsageHarnessTitles: Record<string, string> = {
transcription: "Transcription",
speech: "Speech",
image: "Image generation",
image_edit: "Image editing",
embedding: "Embeddings",
};