Add the harness, envs, per-bot models and chat attachments

- Harness: providers, model catalog synced hourly, automatic free picks,
  cooldowns, tool loop with context trimming, MCP computer tools
- Bots queue runs replies and tasks through the harness; jobs wait
  (attempt refunded) while every model is resting
- Envs: Settings > Environment, scoped general/workspace/computer,
  written to computers and redacted from tool output
- Models: Settings > Models, per-harness provider select defaulting to
  OpenRouter, and a models modal on each bot's chat (the coordinator's
  edits the defaults)
- Chat: thinking row while a reply runs, file attachments (upload,
  paste, drop) that replies read as images, audio and document text
- Fix media uploads failing after a user's first file, and private
  files being served from the public media path

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/harness/audio/looks-like-missing-audio.test.ts

@@ -0,0 +1,27 @@
+import { describe, expect, test } from "bun:test";
+import looksLikeMissingAudio from "./looks-like-missing-audio";
+
+describe("looksLikeMissingAudio", () => {
+    test("spots a model asking for the audio it never got", () => {
+        for (const text of [
+            "Please provide the audio file you’d like transcribed.",
+            "I’m sorry, but I can’t transcribe without the audio.",
+            "I cannot hear any audio in your message.",
+            "There is no audio attached.",
+            "I didn't receive any audio.",
+        ]) {
+            expect(looksLikeMissingAudio({ text })).toBe(true);
+        }
+    });
+
+    test("leaves real transcripts alone", () => {
+        expect(looksLikeMissingAudio({ text: "Hello from cobble." })).toBe(
+            false,
+        );
+        expect(
+            looksLikeMissingAudio({
+                text: `Can you send the audio report to finance? ${"And then more. ".repeat(30)}`,
+            }),
+        ).toBe(false);
+    });
+});

src/harness/audio/looks-like-missing-audio.ts

@@ -0,0 +1,14 @@
+type Params = {
+    text: string;
+};
+
+const MissingAudio =
+    /(provide|upload|share|attach|send)\b.{0,40}\baudio|can(?:'|’)?t (?:transcribe|hear|access|listen)|cannot (?:transcribe|hear|access|listen)|no audio|without (?:the |an |any )?audio|didn(?:'|’)?t (?:receive|get) (?:the |an |any )?audio/i;
+
+/**
+ * Whether a chat model's "transcript" is really it saying it got no audio.
+ * Some providers list audio input for a model but drop the audio.
+ */
+export default function looksLikeMissingAudio({ text }: Params) {
+    return text.length < 300 && MissingAudio.test(text);
+}

src/harness/audio/speak-text.ts

@@ -0,0 +1,73 @@
+import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
+import grabHarnessAdapter from "../providers/grab-harness-adapter";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import withCandidates from "../resolve/with-candidates";
+import type { HarnessCandidate } from "../types";
+
+type Params = {
+    text: string;
+    /**
+     * `mp3` unless set.
+     */
+    format?: string;
+    signal?: AbortSignal;
+    /**
+     * Uses this bot's speech model and voice when it has its own.
+     */
+    bot_id?: number | string;
+};
+
+/**
+ * Reads text aloud with the speech harness. Uses the voice picked in the
+ * speech setting when the model has it, else the model's first voice.
+ */
+export default async function speakText({
+    text,
+    format,
+    signal,
+    bot_id,
+}: Params) {
+    const [candidates, setting] = await Promise.all([
+        resolveHarnessCandidates({ harness: "speech", bot_id }),
+        grabHarnessSetting({ harness: "speech", bot_id }),
+    ]);
+
+    const { result } = await withCandidates({
+        harness: "speech",
+        candidates,
+        run: async (candidate) =>
+            await grabHarnessAdapter({ kind: candidate.provider.kind }).speak({
+                candidate,
+                text,
+                voice: pickVoice({
+                    candidate,
+                    voice:
+                        typeof setting.options.voice == "string"
+                            ? setting.options.voice
+                            : undefined,
+                }),
+                format,
+                signal: signal || new AbortController().signal,
+            }),
+    });
+
+    return result;
+}
+
+function pickVoice({
+    candidate,
+    voice,
+}: {
+    candidate: HarnessCandidate;
+    voice?: string;
+}) {
+    const voices = candidate.model.voices;
+
+    if (voice && (candidate.source == "pinned" || voices?.includes(voice))) {
+        return voice;
+    }
+
+    if (voices?.[0]) return voices[0];
+
+    return candidate.provider.kind == "openai" ? "alloy" : undefined;
+}

src/harness/audio/transcribe-audio.ts

@@ -0,0 +1,116 @@
+import { AppData } from "@/src/data/app-data";
+import { TranscribeAudioPrompt } from "@/src/data/harness-prompts";
+import AppError from "@/src/utils/app-error";
+import HarnessError from "../harness-error";
+import grabHarnessAdapter from "../providers/grab-harness-adapter";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import withCandidates from "../resolve/with-candidates";
+import type { HarnessAudioInput } from "../types";
+import looksLikeMissingAudio from "./looks-like-missing-audio";
+
+/**
+ * A chat model that drops audio is skipped this long.
+ */
+const MissingAudioCooldownMs = 60 * 60 * 1000;
+
+type Params = {
+    audio: HarnessAudioInput;
+    /**
+     * ISO 639-1 code, when known. Helps speech-to-text models.
+     */
+    language?: string;
+    signal?: AbortSignal;
+    /**
+     * Uses this bot's transcription model when it has its own.
+     */
+    bot_id?: number | string;
+};
+
+/**
+ * Turns speech into text with the transcription harness. Speech-to-text
+ * models get the file at `/audio/transcriptions`; chat models that hear
+ * audio are asked for a verbatim transcript.
+ */
+export default async function transcribeAudio({
+    audio,
+    language,
+    signal,
+    bot_id,
+}: Params) {
+    if (audio.data.byteLength > AppData.HarnessMaxAudioBytes) {
+        throw new AppError(`That recording is too long to transcribe.`);
+    }
+
+    const candidates = await resolveHarnessCandidates({
+        harness: "transcription",
+        bot_id,
+    });
+
+    const final_signal = signal || new AbortController().signal;
+
+    const { result } = await withCandidates({
+        harness: "transcription",
+        candidates,
+        run: async (candidate) => {
+            const adapter = grabHarnessAdapter({
+                kind: candidate.provider.kind,
+            });
+
+            const is_speech_to_text =
+                candidate.model.output_modalities.includes("transcription");
+
+            const text = is_speech_to_text
+                ? (
+                      await adapter.transcribe({
+                          candidate,
+                          audio,
+                          language,
+                          signal: final_signal,
+                      })
+                  ).text
+                : (
+                      await adapter.chat({
+                          candidate,
+                          messages: [
+                              {
+                                  role: "user",
+                                  content: [
+                                      { type: "text", text: TranscribeAudioPrompt },
+                                      {
+                                          type: "audio",
+                                          data: Buffer.from(audio.data).toString(
+                                              "base64",
+                                          ),
+                                          format: audio.format,
+                                      },
+                                  ],
+                              },
+                          ],
+                          temperature: 0,
+                          signal: final_signal,
+                      })
+                  ).text;
+
+            if (text === null || text === undefined) {
+                throw new HarnessError({
+                    message: `The model returned no transcript.`,
+                    retryable: true,
+                    candidate_key: candidate.key,
+                });
+            }
+
+            if (!is_speech_to_text && looksLikeMissingAudio({ text })) {
+                throw new HarnessError({
+                    message: `${candidate.model.name || candidate.model.slug} didn't get the audio.`,
+                    retryable: true,
+                    retry_at: Date.now() + MissingAudioCooldownMs,
+                    candidate_key: candidate.key,
+                });
+            }
+
+            return text.trim();
+        },
+    });
+
+    return result;
+}

src/harness/catalog/guess-model-modalities.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, test } from "bun:test";
+import guessModelModalities from "./guess-model-modalities";
+
+const outputs = (slug: string) =>
+    guessModelModalities({ slug }).output_modalities;
+
+describe("guessModelModalities", () => {
+    test("spots speech, transcription, image and embedding models", () => {
+        expect(outputs("whisper-large-v3")).toEqual(["transcription"]);
+        expect(outputs("gpt-4o-mini-transcribe")).toEqual(["transcription"]);
+        expect(outputs("tts-1-hd")).toEqual(["speech"]);
+        expect(outputs("gpt-image-1")).toEqual(["image"]);
+        expect(outputs("dall-e-3")).toEqual(["image"]);
+        expect(outputs("text-embedding-3-small")).toEqual(["embeddings"]);
+    });
+
+    test("takes everything else for a tool-calling chat model", () => {
+        expect(guessModelModalities({ slug: "deepseek-chat" })).toEqual({
+            input_modalities: ["text"],
+            output_modalities: ["text"],
+            supports_tools: true,
+        });
+        expect(outputs("mistral-large-latest")).toEqual(["text"]);
+    });
+
+    test("knows which chat families see images", () => {
+        expect(
+            guessModelModalities({ slug: "claude-sonnet-5-5" }).input_modalities,
+        ).toEqual(["text", "image"]);
+        expect(
+            guessModelModalities({ slug: "models/gemini-3.5-flash" })
+                .input_modalities,
+        ).toEqual(["text", "image"]);
+        expect(
+            guessModelModalities({ slug: "qwen2.5-vl:7b" }).input_modalities,
+        ).toEqual(["text", "image"]);
+        expect(
+            guessModelModalities({ slug: "deepseek-r1:14b" }).input_modalities,
+        ).toEqual(["text"]);
+    });
+});

src/harness/catalog/guess-model-modalities.ts

@@ -0,0 +1,65 @@
+import type { HarnessModelInfo } from "../types";
+
+type Params = {
+    slug: string;
+};
+
+type Guess = Pick<
+    HarnessModelInfo,
+    "input_modalities" | "output_modalities" | "supports_tools"
+>;
+
+const Transcription = /whisper|transcri|(?:^|[-_/])(stt|asr)(?=$|[-_:.])|scribe/i;
+const Speech = /(?:^|[-_/])tts(?=$|[-_:.])|speech|voice/i;
+const ImageOut = /dall-e|gpt-image|imagen|flux|stable-diffusion|sdxl|image-gen/i;
+const Embedding = /embed/i;
+/**
+ * Chat families that read images.
+ */
+const ImageIn =
+    /vision|(?:^|[-_/])vl(?=$|[-_:.])|gpt-4o|gpt-4\.1|gpt-[5-9]|claude|gemini|llava|pixtral|gemma-?[34]|grok-[2-9]|llama-?4|mistral-(small|medium)/i;
+
+/**
+ * What a model takes and makes, guessed from its id, for providers whose
+ * model list only has ids. Anything that isn't plainly a speech, image or
+ * embedding model is taken for a chat model that can call tools.
+ */
+export default function guessModelModalities({ slug }: Params): Guess {
+    if (Transcription.test(slug)) {
+        return {
+            input_modalities: ["audio"],
+            output_modalities: ["transcription"],
+            supports_tools: false,
+        };
+    }
+
+    if (Speech.test(slug)) {
+        return {
+            input_modalities: ["text"],
+            output_modalities: ["speech"],
+            supports_tools: false,
+        };
+    }
+
+    if (ImageOut.test(slug)) {
+        return {
+            input_modalities: ["text", "image"],
+            output_modalities: ["image"],
+            supports_tools: false,
+        };
+    }
+
+    if (Embedding.test(slug)) {
+        return {
+            input_modalities: ["text"],
+            output_modalities: ["embeddings"],
+            supports_tools: false,
+        };
+    }
+
+    return {
+        input_modalities: ImageIn.test(slug) ? ["text", "image"] : ["text"],
+        output_modalities: ["text"],
+        supports_tools: true,
+    };
+}

src/harness/catalog/model-serves-harness.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, test } from "bun:test";
+import modelServesHarness from "./model-serves-harness";
+
+const chat = {
+    input_modalities: ["text"],
+    output_modalities: ["text"],
+    supports_tools: true,
+};
+
+describe("modelServesHarness", () => {
+    test("text needs tools", () => {
+        expect(modelServesHarness({ model: chat, harness: "text" })).toBe(true);
+        expect(
+            modelServesHarness({
+                model: { ...chat, supports_tools: false },
+                harness: "text",
+            }),
+        ).toBe(false);
+    });
+
+    test("vision needs image input", () => {
+        expect(modelServesHarness({ model: chat, harness: "vision" })).toBe(
+            false,
+        );
+        expect(
+            modelServesHarness({
+                model: { ...chat, input_modalities: ["text", "image"] },
+                harness: "vision",
+            }),
+        ).toBe(true);
+    });
+
+    test("transcription takes speech-to-text or audio-hearing chat models", () => {
+        expect(
+            modelServesHarness({
+                model: {
+                    input_modalities: ["audio"],
+                    output_modalities: ["transcription"],
+                    supports_tools: false,
+                },
+                harness: "transcription",
+            }),
+        ).toBe(true);
+        expect(
+            modelServesHarness({
+                model: { ...chat, input_modalities: ["text", "audio"] },
+                harness: "transcription",
+            }),
+        ).toBe(true);
+        expect(
+            modelServesHarness({ model: chat, harness: "transcription" }),
+        ).toBe(false);
+    });
+
+    test("speech and image need that output", () => {
+        expect(
+            modelServesHarness({
+                model: { ...chat, output_modalities: ["speech"] },
+                harness: "speech",
+            }),
+        ).toBe(true);
+        expect(
+            modelServesHarness({
+                model: { ...chat, output_modalities: ["text", "audio"] },
+                harness: "speech",
+            }),
+        ).toBe(false);
+        expect(
+            modelServesHarness({
+                model: { ...chat, output_modalities: ["image", "text"] },
+                harness: "image",
+            }),
+        ).toBe(true);
+    });
+});

src/harness/catalog/model-serves-harness.ts

@@ -0,0 +1,45 @@
+import type { HarnessKind } from "@/src/dict/harness-dict";
+import type { HarnessModelInfo } from "../types";
+
+type Params = {
+    model: Pick<
+        HarnessModelInfo,
+        "input_modalities" | "output_modalities" | "supports_tools"
+    >;
+    harness: HarnessKind;
+};
+
+/**
+ * Whether a model can do a harness's job:
+ * - text: reads and writes text, and calls tools (bots work through tools)
+ * - vision: reads images, writes text
+ * - transcription: a speech-to-text model, or a chat model that hears audio
+ * - speech: a text-to-speech model
+ * - image: makes images
+ */
+export default function modelServesHarness({ model, harness }: Params) {
+    const input = new Set(model.input_modalities);
+    const output = new Set(model.output_modalities);
+
+    switch (harness) {
+        case "text":
+            return (
+                input.has("text") && output.has("text") && model.supports_tools
+            );
+
+        case "vision":
+            return input.has("image") && output.has("text");
+
+        case "transcription":
+            return (
+                output.has("transcription") ||
+                (input.has("audio") && output.has("text"))
+            );
+
+        case "speech":
+            return output.has("speech");
+
+        case "image":
+            return output.has("image");
+    }
+}

src/harness/catalog/parse-model-size.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, test } from "bun:test";
+import parseModelSize from "./parse-model-size";
+
+describe("parseModelSize", () => {
+    test("reads total parameters, skipping active ones", () => {
+        expect(parseModelSize({ slug: "qwen/qwen3-30b-a3b" })).toBe(30);
+        expect(
+            parseModelSize({ slug: "nvidia/nemotron-3-ultra-550b-a55b:free" }),
+        ).toBe(550);
+        expect(parseModelSize({ slug: "liquid/lfm-2.5-2.6b:free" })).toBe(2.6);
+        expect(parseModelSize({ slug: "deepseek-r1:14b" })).toBe(14);
+    });
+
+    test("reads trillions", () => {
+        expect(parseModelSize({ slug: "qwen/qwen3.8-2.4t-a95b" })).toBe(2400);
+    });
+
+    test("is undefined when the slug has no size", () => {
+        expect(parseModelSize({ slug: "deepseek/deepseek-v4.1-flash" })).toBe(
+            undefined,
+        );
+        expect(parseModelSize({ slug: "google/gemma-3n-e4b-it" })).toBe(
+            undefined,
+        );
+    });
+});

src/harness/catalog/parse-model-size.ts

@@ -0,0 +1,19 @@
+type Params = {
+    slug: string;
+};
+
+/**
+ * Total parameters in billions, read from a model slug
+ * (`qwen3-30b-a3b` is 30, `qwen3.8-2.4t-a95b` is 2400). Active-parameter
+ * counts (`a3b`) are skipped. Undefined when the slug doesn't say.
+ */
+export default function parseModelSize({ slug }: Params): number | undefined {
+    const name = slug.split("/").pop() || slug;
+    const match = name.match(/(?<![a-z0-9.])(\d+(?:\.\d+)?)([bt])(?![a-z])/i);
+
+    if (!match?.[1] || !match[2]) return undefined;
+
+    const size = Number(match[1]);
+
+    return match[2].toLowerCase() == "t" ? size * 1000 : size;
+}

src/harness/catalog/rank-auto-models.test.ts

@@ -0,0 +1,165 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessCatalogModel } from "../types";
+import rankAutoModels from "./rank-auto-models";
+
+const day = 24 * 60 * 60 * 1000;
+
+function model(
+    slug: string,
+    extra: Partial<HarnessCatalogModel> = {},
+): HarnessCatalogModel {
+    return {
+        slug,
+        free: true,
+        input_modalities: ["text"],
+        output_modalities: ["text"],
+        supports_tools: true,
+        context_length: 262144,
+        released_at: 0,
+        ...extra,
+    };
+}
+
+/**
+ * OpenRouter's free list on 2026-10-18 (no DeepSeek or Qwen that day).
+ */
+const live = [
+    model("xingchen-agi/xing4.0-29b-a4b:free", { released_at: 20 * day }),
+    model("nvidia/nemotron-3.5-super-vl:free", {
+        input_modalities: ["text", "image", "video"],
+        intelligence_score: 26.5,
+    }),
+    model("liquid/lfm-2.5-2.6b:free", {
+        context_length: 65536,
+        intelligence_score: 8.4,
+    }),
+    model("nvidia/nemotron-3.5-lightning:free", { intelligence_score: 12.9 }),
+    model("thinkingmachines/inkling-small:free", {
+        input_modalities: ["text", "image", "audio"],
+        intelligence_score: 25.7,
+        agentic_score: 23.5,
+    }),
+    model("cohere/north-mini-code:free", { intelligence_score: 9.9 }),
+    model("nvidia/nemotron-3.5-content-safety:free", {
+        supports_tools: false,
+    }),
+    model("nvidia/nemotron-3-ultra-550b-a55b:free", {
+        intelligence_score: 22.9,
+    }),
+    model("google/gemma-4-31b-it:free", {
+        input_modalities: ["image", "text", "video"],
+        intelligence_score: 14.7,
+    }),
+    model("openrouter/free", { input_modalities: ["text", "image"] }),
+    model("anthropic/claude-paid", { free: false }),
+];
+
+const slugs = (models: HarnessCatalogModel[]) => models.map((m) => m.slug);
+
+describe("rankAutoModels", () => {
+    test("follows the vendor preference, then competent others, then routers", () => {
+        expect(slugs(rankAutoModels({ models: live, harness: "text" }))).toEqual(
+            [
+                "nvidia/nemotron-3.5-super-vl:free",
+                "nvidia/nemotron-3-ultra-550b-a55b:free",
+                "nvidia/nemotron-3.5-lightning:free",
+                "liquid/lfm-2.5-2.6b:free",
+                "thinkingmachines/inkling-small:free",
+                "google/gemma-4-31b-it:free",
+                "xingchen-agi/xing4.0-29b-a4b:free",
+                "openrouter/free",
+            ],
+        );
+    });
+
+    test("puts DeepSeek, then Qwen, ahead of everything once they're free", () => {
+        const ranked = rankAutoModels({
+            models: [
+                ...live,
+                model("qwen/qwen3.8-flash:free"),
+                model("deepseek/deepseek-v4.1-flash:free"),
+            ],
+            harness: "text",
+        });
+
+        expect(slugs(ranked).slice(0, 2)).toEqual([
+            "deepseek/deepseek-v4.1-flash:free",
+            "qwen/qwen3.8-flash:free",
+        ]);
+    });
+
+    test("recognises vendors in local model names", () => {
+        const ranked = rankAutoModels({
+            models: [model("llama3.3:70b"), model("qwen3:32b")],
+            harness: "text",
+        });
+
+        expect(slugs(ranked)).toEqual(["qwen3:32b", "llama3.3:70b"]);
+    });
+
+    test("filters by what the harness needs", () => {
+        expect(
+            slugs(rankAutoModels({ models: live, harness: "vision" })),
+        ).toEqual([
+            "nvidia/nemotron-3.5-super-vl:free",
+            "thinkingmachines/inkling-small:free",
+            "google/gemma-4-31b-it:free",
+            "openrouter/free",
+        ]);
+        expect(
+            slugs(rankAutoModels({ models: live, harness: "transcription" })),
+        ).toEqual(["thinkingmachines/inkling-small:free"]);
+        expect(rankAutoModels({ models: live, harness: "speech" })).toEqual(
+            [],
+        );
+    });
+
+    test("puts speech-to-text models ahead of chat models that hear audio", () => {
+        const ranked = rankAutoModels({
+            models: [
+                model("deepseek/omni", { input_modalities: ["text", "audio"] }),
+                model("acme/scribe", {
+                    input_modalities: ["audio"],
+                    output_modalities: ["transcription"],
+                    supports_tools: false,
+                }),
+            ],
+            harness: "transcription",
+        });
+
+        expect(slugs(ranked)).toEqual(["acme/scribe", "deepseek/omni"]);
+    });
+
+    test("drops retired and unscored small models from other vendors", () => {
+        const ranked = rankAutoModels({
+            models: [
+                model("acme/tiny-7b"),
+                model("acme/big-mini"),
+                model("acme/gone", { expires_at: 10 }),
+                model("acme/short", { context_length: 8000 }),
+                model("acme/solid-70b"),
+            ],
+            harness: "text",
+            now: 100,
+        });
+
+        expect(slugs(ranked)).toEqual(["acme/solid-70b"]);
+    });
+
+    test("within a vendor, unscored full-size models beat small ones, newest first", () => {
+        const ranked = rankAutoModels({
+            models: [
+                model("deepseek/deepseek-lite", { released_at: 3 }),
+                model("deepseek/deepseek-old", { released_at: 1 }),
+                model("deepseek/deepseek-new", { released_at: 2 }),
+            ],
+            harness: "text",
+        });
+
+        expect(slugs(ranked)).toEqual([
+            "deepseek/deepseek-new",
+            "deepseek/deepseek-old",
+            "deepseek/deepseek-lite",
+        ]);
+    });
+});

src/harness/catalog/rank-auto-models.ts

@@ -0,0 +1,163 @@
+import { AppData } from "@/src/data/app-data";
+import {
+    AutoModelRouterSlugs,
+    AutoModelVendorPreference,
+    SmallModelPattern,
+    SpecialistModelPattern,
+    type HarnessKind,
+} from "@/src/dict/harness-dict";
+import type { HarnessCatalogModel } from "../types";
+import modelServesHarness from "./model-serves-harness";
+import parseModelSize from "./parse-model-size";
+
+type Params<T extends HarnessCatalogModel> = {
+    models: T[];
+    harness: HarnessKind;
+    /**
+     * Unix ms, for retired models. Defaults to now.
+     */
+    now?: number;
+};
+
+/**
+ * The free models a harness picks automatically, best first:
+ * 1. `AutoModelVendorPreference` in order (DeepSeek, Qwen, NVIDIA, Liquid)
+ * 2. any other vendor's model that looks competent (text and vision only)
+ * 3. routers such as `openrouter/free`, last
+ *
+ * Within a group, benchmarked models come first by intelligence index,
+ * then the rest: full-size before small variants, newest first.
+ */
+export default function rankAutoModels<T extends HarnessCatalogModel>({
+    models,
+    harness,
+    now = Date.now(),
+}: Params<T>): T[] {
+    const ranked = models
+        .filter(
+            (model) =>
+                model.free &&
+                !(model.expires_at && model.expires_at <= now) &&
+                !SpecialistModelPattern.test(model.slug) &&
+                modelServesHarness({ model, harness }),
+        )
+        .map((model) => ({ model, tier: grabTier({ model, harness }) }))
+        .filter(
+            (entry): entry is { model: T; tier: number } =>
+                entry.tier !== undefined,
+        );
+
+    return ranked
+        .sort(
+            (a, b) =>
+                (harness == "transcription"
+                    ? Number(!isSpeechToText(a.model)) -
+                      Number(!isSpeechToText(b.model))
+                    : 0) ||
+                a.tier - b.tier ||
+                compareInTier(a.model, b.model),
+        )
+        .map((entry) => entry.model);
+}
+
+/**
+ * Dedicated speech-to-text models transcribe before chat models that hear
+ * audio: some free chat endpoints list audio input but drop it.
+ */
+function isSpeechToText(model: HarnessCatalogModel) {
+    return model.output_modalities.includes("transcription");
+}
+
+const OtherTier = AutoModelVendorPreference.length;
+const RouterTier = OtherTier + 1;
+
+/**
+ * Undefined drops the model.
+ */
+function grabTier({
+    model,
+    harness,
+}: {
+    model: HarnessCatalogModel;
+    harness: HarnessKind;
+}) {
+    const slug = model.slug.toLowerCase();
+
+    if (AutoModelRouterSlugs.includes(slug)) {
+        return RouterTier;
+    }
+
+    const name = slug.split("/").pop() || slug;
+    const vendor_tier = AutoModelVendorPreference.findIndex(
+        (vendor) => slug.startsWith(`${vendor}/`) || name.startsWith(vendor),
+    );
+
+    if (vendor_tier >= 0) {
+        return vendor_tier;
+    }
+
+    if (harness != "text" && harness != "vision") {
+        return OtherTier;
+    }
+
+    return isCompetent({ model }) ? OtherTier : undefined;
+}
+
+function isCompetent({ model }: { model: HarnessCatalogModel }) {
+    if (
+        model.context_length &&
+        model.context_length < AppData.HarnessAutoMinContext
+    ) {
+        return false;
+    }
+
+    if (model.intelligence_score !== undefined) {
+        return model.intelligence_score >= AppData.HarnessAutoMinScore;
+    }
+
+    const size = parseModelSize({ slug: model.slug });
+
+    return (
+        !SmallModelPattern.test(model.slug) &&
+        (size === undefined || size >= AppData.HarnessAutoMinSizeB)
+    );
+}
+
+function compareInTier(a: HarnessCatalogModel, b: HarnessCatalogModel) {
+    const a_scored = a.intelligence_score !== undefined;
+    const b_scored = b.intelligence_score !== undefined;
+
+    if (a_scored != b_scored) {
+        return a_scored ? -1 : 1;
+    }
+
+    if (a_scored && b_scored) {
+        const by_score =
+            (b.intelligence_score || 0) - (a.intelligence_score || 0) ||
+            (b.agentic_score || 0) - (a.agentic_score || 0);
+
+        if (by_score) return by_score;
+    }
+
+    const a_small = isSmall(a);
+    const b_small = isSmall(b);
+
+    if (a_small != b_small) {
+        return a_small ? 1 : -1;
+    }
+
+    return (
+        (b.released_at || 0) - (a.released_at || 0) ||
+        (b.context_length || 0) - (a.context_length || 0) ||
+        a.slug.localeCompare(b.slug)
+    );
+}
+
+function isSmall(model: HarnessCatalogModel) {
+    const size = parseModelSize({ slug: model.slug });
+
+    return (
+        SmallModelPattern.test(model.slug) ||
+        (size !== undefined && size < AppData.HarnessAutoMinSizeB)
+    );
+}

src/harness/harness-error.ts

@@ -0,0 +1,66 @@
+/**
+ * - `provider`: the provider refused or failed the call
+ * - `unavailable`: no model is set up (or left) for the harness
+ * - `aborted`: the caller cancelled
+ */
+export type HarnessErrorCode = "provider" | "unavailable" | "aborted";
+
+type Params = {
+    message: string;
+    code?: HarnessErrorCode;
+    status?: number;
+    retryable?: boolean;
+    retry_at?: number;
+    provider_wide?: boolean;
+    candidate_key?: string;
+    resting?: boolean;
+};
+
+/**
+ * A failed harness call. Its message is safe to show the user.
+ */
+export default class HarnessError extends Error {
+    code: HarnessErrorCode;
+    status?: number;
+    /**
+     * Worth trying again later (rate limit, outage, timeout). False when
+     * only a different model or setting can help.
+     */
+    retryable: boolean;
+    /**
+     * Unix ms before which the failed candidate shouldn't be tried again.
+     */
+    retry_at?: number;
+    /**
+     * The provider's key or account is the problem, so every model on it
+     * fails the same way.
+     */
+    provider_wide: boolean;
+    candidate_key?: string;
+    /**
+     * No model was tried: every one is cooling down until `retry_at`. A
+     * queued job waits instead of using up an attempt.
+     */
+    resting: boolean;
+
+    constructor({
+        message,
+        code = "provider",
+        status,
+        retryable = false,
+        retry_at,
+        provider_wide = false,
+        candidate_key,
+        resting = false,
+    }: Params) {
+        super(message);
+        this.name = "HarnessError";
+        this.code = code;
+        this.status = status;
+        this.retryable = retryable;
+        this.retry_at = retry_at;
+        this.provider_wide = provider_wide;
+        this.candidate_key = candidate_key;
+        this.resting = resting;
+    }
+}

src/harness/image/generate-image.ts

@@ -0,0 +1,50 @@
+import grabHarnessAdapter from "../providers/grab-harness-adapter";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import withCandidates from "../resolve/with-candidates";
+
+type Params = {
+    prompt: string;
+    n?: number;
+    size?: string;
+    /**
+     * Reference images: https or `data:` URLs.
+     */
+    references?: string[];
+    signal?: AbortSignal;
+    /**
+     * Uses this bot's image model when it has its own.
+     */
+    bot_id?: number | string;
+};
+
+/**
+ * Makes images with the image generation harness. Returns their bytes.
+ */
+export default async function generateImage({
+    prompt,
+    n,
+    size,
+    references,
+    signal,
+    bot_id,
+}: Params) {
+    const candidates = await resolveHarnessCandidates({ harness: "image", bot_id });
+
+    const { result } = await withCandidates({
+        harness: "image",
+        candidates,
+        run: async (candidate) =>
+            await grabHarnessAdapter({
+                kind: candidate.provider.kind,
+            }).generateImage({
+                candidate,
+                prompt,
+                n,
+                size,
+                references,
+                signal: signal || new AbortController().signal,
+            }),
+    });
+
+    return result.images;
+}

src/harness/mcp/connect-mcp-server.ts

@@ -0,0 +1,58 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { AppData } from "@/src/data/app-data";
+import { SiteData } from "@/src/data/site-data";
+
+type Params = {
+    /**
+     * The server's MCP endpoint, e.g. `http://cobble-computer:8080/mcp`.
+     */
+    url: string;
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+};
+
+/**
+ * Connects to an MCP server over Streamable HTTP, falling back to the
+ * older SSE transport. Close the client when the run is done.
+ */
+export default async function connectMcpServer({
+    url,
+    headers,
+    signal,
+}: Params): Promise<Client> {
+    const endpoint = new URL(url);
+    const request_init = headers ? { headers } : undefined;
+    const options = { signal, timeout: AppData.McpConnectTimeoutMs };
+
+    try {
+        const client = newClient();
+        await client.connect(
+            new StreamableHTTPClientTransport(endpoint, { requestInit: request_init }),
+            options,
+        );
+        return client;
+    } catch (error: unknown) {
+        if (signal?.aborted) throw error;
+
+        try {
+            const client = newClient();
+            await client.connect(
+                new SSEClientTransport(endpoint, { requestInit: request_init }),
+                options,
+            );
+            return client;
+        } catch {
+            throw new Error(
+                `Couldn't connect to the MCP server at ${endpoint.origin}: ${
+                    error instanceof Error ? error.message : String(error)
+                }`,
+            );
+        }
+    }
+}
+
+function newClient() {
+    return new Client({ name: SiteData["SiteSlug"], version: "1.0.0" });
+}

src/harness/mcp/grab-mcp-tools.ts

@@ -0,0 +1,74 @@
+import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { AppData } from "@/src/data/app-data";
+import type { HarnessTool } from "../types";
+import mcpResultToText, { type McpImage } from "./mcp-result-to-text";
+
+type Params = {
+    client: Client;
+    /**
+     * Tool names to leave out.
+     */
+    exclude?: readonly string[];
+    describeImage?: (image: McpImage) => Promise<string>;
+    /**
+     * Applied to every result (and error) before the model sees it, e.g.
+     * to hide secret values.
+     */
+    redact?: (text: string) => string;
+};
+
+/**
+ * Every tool an MCP server offers, as harness tools. Arguments go to the
+ * server as the model wrote them (the server checks them against its own
+ * schema); results come back as text via `mcpResultToText`.
+ */
+export default async function grabMcpTools({
+    client,
+    exclude = [],
+    describeImage,
+    redact = (text) => text,
+}: Params): Promise<HarnessTool[]> {
+    const tools: HarnessTool[] = [];
+    let cursor: string | undefined;
+
+    do {
+        const page = await client.listTools(cursor ? { cursor } : undefined, {
+            timeout: AppData.McpConnectTimeoutMs,
+        });
+
+        for (const tool of page.tools) {
+            if (exclude.includes(tool.name)) continue;
+
+            tools.push({
+                name: tool.name,
+                description: tool.description || tool.title || tool.name,
+                parameters: tool.inputSchema as Record<string, unknown>,
+                run: async ({ args, signal }) => {
+                    try {
+                        return redact(
+                            await mcpResultToText({
+                                result: await client.callTool(
+                                    { name: tool.name, arguments: args },
+                                    undefined,
+                                    {
+                                        signal,
+                                        timeout: AppData.McpToolTimeoutMs,
+                                        resetTimeoutOnProgress: true,
+                                    },
+                                ),
+                                describeImage,
+                            }),
+                        );
+                    } catch (error: unknown) {
+                        if (error instanceof Error) error.message = redact(error.message);
+                        throw error;
+                    }
+                },
+            });
+        }
+
+        cursor = page.nextCursor;
+    } while (cursor);
+
+    return tools;
+}

src/harness/mcp/mcp-result-to-text.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, test } from "bun:test";
+import mcpResultToText from "./mcp-result-to-text";
+
+describe("mcpResultToText", () => {
+    test("joins text parts", async () => {
+        expect(
+            await mcpResultToText({
+                result: {
+                    content: [
+                        { type: "text", text: "one" },
+                        { type: "text", text: "two" },
+                    ],
+                },
+            }),
+        ).toBe("one\n\ntwo");
+    });
+
+    test("describes images with the given reader", async () => {
+        expect(
+            await mcpResultToText({
+                result: {
+                    content: [{ type: "image", data: "aGk=", mimeType: "image/png" }],
+                },
+                describeImage: async ({ data, mime_type }) =>
+                    `${mime_type} ${data}`,
+            }),
+        ).toBe("[Image: image/png aGk=]");
+    });
+
+    test("notes images when nothing can read them", async () => {
+        expect(
+            await mcpResultToText({
+                result: {
+                    content: [{ type: "image", data: "aGk=", mimeType: "image/png" }],
+                },
+            }),
+        ).toBe("[Image (image/png)]");
+    });
+
+    test("keeps going when an image can't be read", async () => {
+        expect(
+            await mcpResultToText({
+                result: { content: [{ type: "image", data: "aGk=" }] },
+                describeImage: async () => {
+                    throw new Error("No vision model.");
+                },
+            }),
+        ).toBe("[Image: couldn't be read. No vision model.]");
+    });
+
+    test("falls back to structured content", async () => {
+        expect(
+            await mcpResultToText({
+                result: { content: [], structuredContent: { ok: true } },
+            }),
+        ).toBe(`{"ok":true}`);
+    });
+
+    test("throws error results", async () => {
+        expect(
+            mcpResultToText({
+                result: {
+                    isError: true,
+                    content: [{ type: "text", text: "No such file" }],
+                },
+            }),
+        ).rejects.toThrow("No such file");
+    });
+
+    test("says when the tool returned nothing", async () => {
+        expect(await mcpResultToText({ result: {} })).toBe(
+            "Done. The tool returned nothing.",
+        );
+    });
+});

src/harness/mcp/mcp-result-to-text.ts

@@ -0,0 +1,101 @@
+export type McpImage = {
+    /**
+     * Base64.
+     */
+    data: string;
+    mime_type: string;
+};
+
+type Params = {
+    /**
+     * A `tools/call` result.
+     */
+    result: {
+        content?: unknown;
+        structuredContent?: unknown;
+        isError?: unknown;
+        /**
+         * Old servers (protocol 2024-10-07) answer with this instead.
+         */
+        toolResult?: unknown;
+    };
+    /**
+     * Turns an image (a screenshot) into text for the model. Without it,
+     * images are noted but not described.
+     */
+    describeImage?: (image: McpImage) => Promise<string>;
+};
+
+/**
+ * An MCP tool result as text for the model: text as is, images described,
+ * resources by their text or link. Tool messages carry text only, so an
+ * image reaches the model as its reading. A result flagged as an error is
+ * thrown, so the model gets it as `Error: ...`.
+ */
+export default async function mcpResultToText({
+    result,
+    describeImage,
+}: Params): Promise<string> {
+    const parts = Array.isArray(result.content) ? result.content : [];
+    const texts: string[] = [];
+
+    for (const part of parts) {
+        texts.push(await partToText({ part, describeImage }));
+    }
+
+    const structured = result.structuredContent ?? result.toolResult;
+
+    if (!texts.length && structured !== undefined) {
+        texts.push(typeof structured == "string" ? structured : JSON.stringify(structured));
+    }
+
+    const text = texts.filter(Boolean).join("\n\n") || "Done. The tool returned nothing.";
+
+    if (result.isError) {
+        throw new Error(text);
+    }
+
+    return text;
+}
+
+async function partToText({
+    part,
+    describeImage,
+}: {
+    part: any;
+    describeImage?: Params["describeImage"];
+}): Promise<string> {
+    switch (part?.type) {
+        case "text":
+            return String(part.text ?? "");
+
+        case "image": {
+            if (!describeImage || typeof part.data != "string") {
+                return `[Image (${part.mimeType || "unknown type"})]`;
+            }
+
+            try {
+                return `[Image: ${await describeImage({
+                    data: part.data,
+                    mime_type: part.mimeType || "image/png",
+                })}]`;
+            } catch (error: unknown) {
+                return `[Image: couldn't be read. ${error instanceof Error ? error.message : String(error)}]`;
+            }
+        }
+
+        case "audio":
+            return `[Audio (${part.mimeType || "unknown type"})]`;
+
+        case "resource":
+            return typeof part.resource?.text == "string"
+                ? part.resource.text
+                : `[File: ${part.resource?.uri || "unnamed"}]`;
+
+        case "resource_link":
+            return `[Link: ${part.name || part.uri} ${part.uri || ""}]`.trim();
+
+        default:
+            return "";
+    }
+}

src/harness/media/read-image.ts

@@ -0,0 +1,72 @@
+import { ReadImagePrompt } from "@/src/data/harness-prompts";
+import HarnessError from "../harness-error";
+import grabHarnessAdapter from "../providers/grab-harness-adapter";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import withCandidates from "../resolve/with-candidates";
+
+type Params = {
+    /**
+     * An https or `data:` URL.
+     */
+    url: string;
+    /**
+     * What to look for, added to the standard reading prompt.
+     */
+    focus?: string;
+    signal?: AbortSignal;
+    /**
+     * Uses this bot's media reading model when it has its own.
+     */
+    bot_id?: number | string;
+};
+
+/**
+ * Reads an image with the media reading (vision) harness and returns what
+ * it sees, as text for a model that can't see.
+ */
+export default async function readImage({ url, focus, signal, bot_id }: Params) {
+    const candidates = await resolveHarnessCandidates({
+        harness: "vision",
+        bot_id,
+    });
+
+    const { result } = await withCandidates({
+        harness: "vision",
+        candidates,
+        run: async (candidate) => {
+            const answer = await grabHarnessAdapter({
+                kind: candidate.provider.kind,
+            }).chat({
+                candidate,
+                messages: [
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "text",
+                                text: focus
+                                    ? `${ReadImagePrompt}\n\nPay special attention to: ${focus}`
+                                    : ReadImagePrompt,
+                            },
+                            { type: "image", url },
+                        ],
+                    },
+                ],
+                temperature: 0,
+                signal: signal || new AbortController().signal,
+            });
+
+            if (!answer.text?.trim()) {
+                throw new HarnessError({
+                    message: `The model couldn't read the image.`,
+                    retryable: true,
+                    candidate_key: candidate.key,
+                });
+            }
+
+            return answer.text.trim();
+        },
+    });
+
+    return result;
+}

src/harness/media/read-media.ts

@@ -0,0 +1,97 @@
+import fileToText from "@/src/functions/backend/db/users/media/file-to-text";
+import AppError from "@/src/utils/app-error";
+import transcribeAudio from "../audio/transcribe-audio";
+import readImage from "./read-image";
+
+type Params = {
+    data: Uint8Array;
+    /**
+     * e.g. `image/png`, `audio/webm`, `application/pdf`.
+     */
+    media_type: string;
+    /**
+     * File name, used for its extension when the type is vague.
+     */
+    name?: string;
+    signal?: AbortSignal;
+    /**
+     * Reads with this bot's models when it has its own.
+     */
+    bot_id?: number | string;
+};
+
+const DocumentExtensions = ["pdf", "docx", "txt", "md", "csv", "json", "html", "xml", "yaml", "yml"];
+
+const TypeExtensions: Record<string, string> = {
+    "application/pdf": "pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        "docx",
+    "text/plain": "txt",
+    "text/markdown": "md",
+    "text/csv": "csv",
+    "application/json": "json",
+};
+
+/**
+ * Reads any attachment into text for the text harness: documents are
+ * extracted on this machine, images go to the media reading harness and
+ * audio to the transcription harness.
+ */
+export default async function readMedia({
+    data,
+    media_type,
+    name,
+    signal,
+    bot_id,
+}: Params): Promise<string> {
+    const type = media_type.toLowerCase().split(";")[0]!.trim();
+    const ext = (
+        name?.includes(".")
+            ? name.split(".").pop()
+            : TypeExtensions[type] || type.split("/").pop()
+    )?.toLowerCase() || "";
+
+    if (type.startsWith("image/")) {
+        return await readImage({
+            url: `data:${type};base64,${Buffer.from(data).toString("base64")}`,
+            signal,
+            bot_id,
+        });
+    }
+
+    if (type.startsWith("audio/")) {
+        return await transcribeAudio({
+            audio: { data, format: audioFormat({ type, ext }) },
+            signal,
+            bot_id,
+        });
+    }
+
+    if (type.startsWith("text/") || DocumentExtensions.includes(ext)) {
+        const text = await fileToText({
+            file_stream: Buffer.from(data),
+            file_type: ext,
+        });
+
+        if (!text?.trim()) {
+            throw new AppError(
+                ext == "pdf"
+                    ? `This PDF has no text in it (it may be a scan). Send its pages as images instead.`
+                    : `This file has no text in it.`,
+            );
+        }
+
+        return text;
+    }
+
+    throw new AppError(`Bots can't read ${ext ? `.${ext}` : "this kind of"} files yet.`);
+}
+
+function audioFormat({ type, ext }: { type: string; ext: string }) {
+    if (type.includes("mpeg") || ext == "mp3") return "mp3";
+    if (type.includes("mp4") || ext == "m4a") return "m4a";
+    if (type.includes("wav")) return "wav";
+    if (type.includes("ogg")) return "ogg";
+    if (type.includes("flac")) return "flac";
+    return ext || "webm";
+}

src/harness/providers/grab-harness-adapter.ts

@@ -0,0 +1,18 @@
+import type { HarnessProviderKind } from "@/src/dict/harness-provider-dict";
+import type { HarnessAdapter } from "../types";
+import OpenAICompatibleAdapter from "./openai-compatible";
+
+type Params = {
+    kind: HarnessProviderKind;
+};
+
+/**
+ * The adapter for a provider kind. Every kind speaks the OpenAI-compatible
+ * API today; a provider that gets a native adapter returns it here.
+ */
+export default function grabHarnessAdapter({ kind }: Params): HarnessAdapter {
+    switch (kind) {
+        default:
+            return OpenAICompatibleAdapter;
+    }
+}

src/harness/providers/grab-provider-headers.ts

@@ -0,0 +1,35 @@
+import { SiteData } from "@/src/data/site-data";
+import type { HarnessProviderConfig } from "../types";
+
+type Params = {
+    provider: HarnessProviderConfig;
+};
+
+/**
+ * Auth and attribution headers for a provider call.
+ */
+export default function grabProviderHeaders({
+    provider,
+}: Params): Record<string, string> {
+    const headers: Record<string, string> = {};
+
+    if (provider.api_key) {
+        headers["Authorization"] = `Bearer ${provider.api_key}`;
+    }
+
+    switch (provider.kind) {
+        case "openrouter":
+            headers["HTTP-Referer"] = SiteData["SiteURL"];
+            headers["X-Title"] = SiteData["SiteName"];
+            break;
+
+        case "anthropic":
+            if (provider.api_key) {
+                headers["x-api-key"] = provider.api_key;
+            }
+            headers["anthropic-version"] = "2023-06-01";
+            break;
+    }
+
+    return headers;
+}

src/harness/providers/grab-provider-preset.ts

@@ -0,0 +1,22 @@
+import {
+    HarnessProviderKinds,
+    type HarnessProviderKind,
+    type HarnessProviderPreset,
+} from "@/src/dict/harness-provider-dict";
+
+type Params = {
+    kind: HarnessProviderKind | string | null | undefined;
+};
+
+/**
+ * The preset for a provider kind. Unknown kinds are treated as a generic
+ * OpenAI-compatible server.
+ */
+export default function grabProviderPreset({
+    kind,
+}: Params): HarnessProviderPreset {
+    return (
+        HarnessProviderKinds.find((preset) => preset.value == kind) ||
+        HarnessProviderKinds.find((preset) => preset.value == "custom")!
+    );
+}

src/harness/providers/openai-compatible/chat.ts

@@ -0,0 +1,49 @@
+import type { HarnessAdapter } from "../../types";
+import grabProviderPreset from "../grab-provider-preset";
+import providerRequest from "./provider-request";
+import readChatResponse from "./read-chat-response";
+import toWireMessages from "./to-wire-messages";
+
+/**
+ * `POST /chat/completions`, with tools when given.
+ */
+const chat: HarnessAdapter["chat"] = async ({
+    candidate,
+    messages,
+    tools,
+    max_tokens,
+    temperature,
+    signal,
+}) => {
+    const { provider, model } = candidate;
+
+    const res = await providerRequest({
+        provider,
+        path: "/chat/completions",
+        json: {
+            model: model.slug,
+            messages: toWireMessages({ messages }),
+            ...(tools?.length
+                ? {
+                      tools: tools.map((tool) => ({
+                          type: "function",
+                          function: tool,
+                      })),
+                  }
+                : {}),
+            ...(max_tokens ? { max_tokens } : {}),
+            ...(temperature !== undefined ? { temperature } : {}),
+        },
+        signal,
+        candidate_key: candidate.key,
+    });
+
+    return readChatResponse({
+        json: await res.json().catch(() => null),
+        provider_name:
+            provider.name || grabProviderPreset({ kind: provider.kind }).title,
+        candidate_key: candidate.key,
+    });
+};
+
+export default chat;

src/harness/providers/openai-compatible/generate-image.ts

@@ -0,0 +1,81 @@
+import HarnessError from "../../harness-error";
+import type { HarnessAdapter, HarnessImageOutput } from "../../types";
+import grabProviderPreset from "../grab-provider-preset";
+import providerRequest from "./provider-request";
+
+/**
+ * Image generation at the provider's `image_path` (`/images` on
+ * OpenRouter, `/images/generations` elsewhere). Images come back as
+ * base64, or as URLs that are downloaded here.
+ */
+const generateImage: HarnessAdapter["generateImage"] = async ({
+    candidate,
+    prompt,
+    n,
+    size,
+    references,
+    signal,
+}) => {
+    const { provider, model } = candidate;
+    const preset = grabProviderPreset({ kind: provider.kind });
+    const is_openrouter = provider.kind == "openrouter";
+
+    const res = await providerRequest({
+        provider,
+        path: preset.image_path,
+        json: {
+            model: model.slug,
+            prompt,
+            ...(n ? { n } : {}),
+            ...(size ? { size } : {}),
+            ...(is_openrouter
+                ? references?.length
+                    ? {
+                          input_references: references.map((url) => ({
+                              type: "image_url",
+                              image_url: { url },
+                          })),
+                      }
+                    : {}
+                : { response_format: "b64_json" }),
+        },
+        signal,
+        candidate_key: candidate.key,
+    });
+
+    const json = await res.json().catch(() => null);
+    const items: any[] = Array.isArray(json?.data) ? json.data : [];
+    const images: HarnessImageOutput[] = [];
+
+    for (const item of items) {
+        if (typeof item?.b64_json == "string") {
+            images.push({
+                data: Uint8Array.from(Buffer.from(item.b64_json, "base64")),
+                media_type: item.media_type || "image/png",
+            });
+        } else if (typeof item?.url == "string") {
+            const download = await fetch(item.url, { signal });
+
+            if (download.ok) {
+                images.push({
+                    data: new Uint8Array(await download.arrayBuffer()),
+                    media_type:
+                        download.headers.get("content-type")?.split(";")[0] ||
+                        "image/png",
+                });
+            }
+        }
+    }
+
+    if (!images.length) {
+        throw new HarnessError({
+            message: `${provider.name || preset.title}: No image came back.`,
+            retryable: true,
+            candidate_key: candidate.key,
+        });
+    }
+
+    return { images };
+};
+
+export default generateImage;

src/harness/providers/openai-compatible/index.ts

@@ -0,0 +1,21 @@
+import type { HarnessAdapter } from "../../types";
+import chat from "./chat";
+import generateImage from "./generate-image";
+import listModels from "./list-models";
+import speak from "./speak";
+import transcribe from "./transcribe";
+
+/**
+ * The OpenAI-compatible API: OpenRouter, OpenAI, Groq, Mistral, xAI,
+ * DeepSeek, Gemini and Anthropic (through their compatibility layers),
+ * Ollama, vLLM, LM Studio and llama.cpp.
+ */
+const OpenAICompatibleAdapter: HarnessAdapter = {
+    chat,
+    transcribe,
+    speak,
+    generateImage,
+    listModels,
+};
+
+export default OpenAICompatibleAdapter;

src/harness/providers/openai-compatible/list-models.ts

@@ -0,0 +1,57 @@
+import guessModelModalities from "../../catalog/guess-model-modalities";
+import type { HarnessAdapter, HarnessCatalogModel } from "../../types";
+import grabProviderPreset from "../grab-provider-preset";
+import openRouterModelToCatalog, {
+    type OpenRouterModel,
+} from "../openrouter/openrouter-model-to-catalog";
+import providerRequest from "./provider-request";
+
+/**
+ * The provider's models. OpenRouter's list is complete (every output
+ * modality, with capabilities and benchmarks); other providers list ids,
+ * and capabilities are guessed from them. A local provider's models are
+ * all free.
+ */
+const listModels: HarnessAdapter["listModels"] = async ({
+    provider,
+    signal,
+}) => {
+    const preset = grabProviderPreset({ kind: provider.kind });
+    const is_openrouter = preset.catalog == "openrouter";
+
+    const res = await providerRequest({
+        provider,
+        path: is_openrouter ? "/models?output_modalities=all" : "/models",
+        method: "GET",
+        signal,
+    });
+
+    const json = await res.json().catch(() => null);
+    const items: any[] = Array.isArray(json?.data)
+        ? json.data
+        : Array.isArray(json?.models)
+          ? json.models
+          : [];
+
+    if (is_openrouter) {
+        return items
+            .filter((item) => typeof item?.id == "string")
+            .map((item: OpenRouterModel) =>
+                openRouterModelToCatalog({ model: item }),
+            );
+    }
+
+    return items
+        .map((item) => (typeof item == "string" ? item : item?.id))
+        .filter((slug): slug is string => typeof slug == "string" && !!slug)
+        .map(
+            (slug): HarnessCatalogModel => ({
+                slug,
+                name: slug,
+                free: preset.local,
+                ...guessModelModalities({ slug }),
+            }),
+        );
+};
+
+export default listModels;

src/harness/providers/openai-compatible/provider-request.ts

@@ -0,0 +1,84 @@
+import { AppData } from "@/src/data/app-data";
+import HarnessError from "../../harness-error";
+import type { HarnessProviderConfig } from "../../types";
+import grabProviderHeaders from "../grab-provider-headers";
+import grabProviderPreset from "../grab-provider-preset";
+import toHarnessError from "../to-harness-error";
+
+type Params = {
+    provider: HarnessProviderConfig;
+    /**
+     * Appended to the provider's base URL, e.g. `/chat/completions`.
+     */
+    path: string;
+    method?: "GET" | "POST";
+    json?: unknown;
+    form?: FormData;
+    signal?: AbortSignal;
+    timeout_ms?: number;
+    candidate_key?: string;
+};
+
+/**
+ * One HTTP call to a provider. Resolves with the response when it's OK;
+ * otherwise throws a `HarnessError` (`aborted` when the caller cancelled,
+ * retryable for timeouts and network failures).
+ */
+export default async function providerRequest({
+    provider,
+    path,
+    method = "POST",
+    json,
+    form,
+    signal,
+    timeout_ms = AppData.HarnessRequestTimeoutMs,
+    candidate_key,
+}: Params): Promise<Response> {
+    const provider_name =
+        provider.name || grabProviderPreset({ kind: provider.kind }).title;
+    const timeout = AbortSignal.timeout(timeout_ms);
+    const headers = grabProviderHeaders({ provider });
+
+    if (json !== undefined) {
+        headers["Content-Type"] = "application/json";
+    }
+
+    let res: Response;
+
+    try {
+        res = await fetch(`${provider.base_url.replace(/\/+$/, "")}${path}`, {
+            method,
+            headers,
+            body: form || (json !== undefined ? JSON.stringify(json) : undefined),
+            signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+        });
+    } catch (error: unknown) {
+        if (signal?.aborted) {
+            throw new HarnessError({
+                message: `Cancelled.`,
+                code: "aborted",
+                candidate_key,
+            });
+        }
+
+        throw new HarnessError({
+            message: timeout.aborted
+                ? `${provider_name} took too long to answer.`
+                : `Couldn't reach ${provider_name}: ${error instanceof Error ? error.message : String(error)}`,
+            retryable: true,
+            candidate_key,
+        });
+    }
+
+    if (!res.ok) {
+        throw toHarnessError({
+            status: res.status,
+            body: await res.text().catch(() => ""),
+            headers: res.headers,
+            provider_name,
+            candidate_key,
+        });
+    }
+
+    return res;
+}

src/harness/providers/openai-compatible/read-chat-response.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, test } from "bun:test";
+import HarnessError from "../../harness-error";
+import readChatResponse from "./read-chat-response";
+
+const read = (json: unknown) =>
+    readChatResponse({ json, provider_name: "OpenRouter" });
+
+describe("readChatResponse", () => {
+    test("reads text, tool calls and usage", () => {
+        expect(
+            read({
+                choices: [
+                    {
+                        finish_reason: "tool_calls",
+                        message: {
+                            content: "Checking.",
+                            tool_calls: [
+                                {
+                                    id: "a1",
+                                    type: "function",
+                                    function: {
+                                        name: "update_memory",
+                                        arguments: `{"memory":"x"}`,
+                                    },
+                                },
+                                {
+                                    function: {
+                                        name: "noop",
+                                        arguments: { a: 1 },
+                                    },
+                                },
+                                { function: {} },
+                            ],
+                        },
+                    },
+                ],
+                usage: { prompt_tokens: 10, completion_tokens: 2, cost: 0 },
+            }),
+        ).toEqual({
+            text: "Checking.",
+            tool_calls: [
+                {
+                    id: "a1",
+                    name: "update_memory",
+                    arguments: `{"memory":"x"}`,
+                },
+                { id: "call_1", name: "noop", arguments: `{"a":1}` },
+            ],
+            finish_reason: "tool_calls",
+            usage: { prompt_tokens: 10, completion_tokens: 2, cost: 0 },
+        });
+    });
+
+    test("joins content parts and treats empty text as none", () => {
+        expect(
+            read({
+                choices: [
+                    {
+                        message: {
+                            content: [
+                                { type: "text", text: "a" },
+                                { type: "text", text: "b" },
+                            ],
+                        },
+                    },
+                ],
+            }).text,
+        ).toBe("ab");
+        expect(read({ choices: [{ message: { content: "" } }] }).text).toBe(
+            null,
+        );
+    });
+
+    test("throws on an error body or no choices", () => {
+        expect(() =>
+            read({ error: { message: "Upstream down", code: 502 } }),
+        ).toThrow(HarnessError);
+        expect(() => read({ choices: [] })).toThrow("returned no answer");
+
+        try {
+            read({ error: { message: "bad", code: 400 } });
+        } catch (error: any) {
+            expect(error.retryable).toBe(false);
+        }
+    });
+});

src/harness/providers/openai-compatible/read-chat-response.ts

@@ -0,0 +1,85 @@
+import HarnessError from "../../harness-error";
+import type { HarnessChatResult, HarnessToolCall } from "../../types";
+
+type WireToolCall = {
+    id?: string;
+    function?: { name?: string; arguments?: string | object };
+};
+
+type Params = {
+    /**
+     * The parsed chat completions response.
+     */
+    json: any;
+    provider_name: string;
+    candidate_key?: string;
+};
+
+/**
+ * Reads a chat completions response. Providers sometimes answer 200 with an
+ * `error` or with no choices; both throw a retryable `HarnessError`.
+ */
+export default function readChatResponse({
+    json,
+    provider_name,
+    candidate_key,
+}: Params): HarnessChatResult {
+    if (json?.error) {
+        const status = Number(json.error.code) || undefined;
+
+        throw new HarnessError({
+            message: `${provider_name}: ${json.error.message || "The model failed."}`,
+            status,
+            retryable: !status || status == 429 || status >= 500,
+            candidate_key,
+        });
+    }
+
+    const choice = json?.choices?.[0];
+
+    if (!choice?.message) {
+        throw new HarnessError({
+            message: `${provider_name}: The model returned no answer.`,
+            retryable: true,
+            candidate_key,
+        });
+    }
+
+    const content = choice.message.content;
+    const text =
+        typeof content == "string"
+            ? content
+            : Array.isArray(content)
+              ? content
+                    .map((part: any) =>
+                        typeof part?.text == "string" ? part.text : "",
+                    )
+                    .join("")
+              : null;
+
+    const tool_calls: HarnessToolCall[] = (
+        (choice.message.tool_calls || []) as WireToolCall[]
+    )
+        .filter((call) => call.function?.name)
+        .map((call, index) => ({
+            id: call.id || `call_${index}`,
+            name: call.function!.name!,
+            arguments:
+                typeof call.function!.arguments == "string"
+                    ? call.function!.arguments
+                    : JSON.stringify(call.function!.arguments || {}),
+        }));
+
+    return {
+        text: text || null,
+        tool_calls,
+        finish_reason: choice.finish_reason || undefined,
+        usage: json.usage
+            ? {
+                  prompt_tokens: json.usage.prompt_tokens,
+                  completion_tokens: json.usage.completion_tokens,
+                  cost: json.usage.cost,
+              }
+            : undefined,
+    };
+}

src/harness/providers/openai-compatible/speak.ts

@@ -0,0 +1,35 @@
+import type { HarnessAdapter } from "../../types";
+import providerRequest from "./provider-request";
+
+/**
+ * `POST /audio/speech`. Returns the audio bytes.
+ */
+const speak: HarnessAdapter["speak"] = async ({
+    candidate,
+    text,
+    voice,
+    format = "mp3",
+    signal,
+}) => {
+    const res = await providerRequest({
+        provider: candidate.provider,
+        path: "/audio/speech",
+        json: {
+            model: candidate.model.slug,
+            input: text,
+            ...(voice ? { voice } : {}),
+            response_format: format,
+        },
+        signal,
+        candidate_key: candidate.key,
+    });
+
+    return {
+        data: new Uint8Array(await res.arrayBuffer()),
+        media_type:
+            res.headers.get("content-type")?.split(";")[0] ||
+            `audio/${format == "mp3" ? "mpeg" : format}`,
+    };
+};
+
+export default speak;

src/harness/providers/openai-compatible/to-wire-messages.ts

@@ -0,0 +1,69 @@
+import type { HarnessContentPart, HarnessMessage } from "../../types";
+
+type Params = {
+    messages: HarnessMessage[];
+};
+
+/**
+ * Harness messages in the OpenAI chat completions format.
+ */
+export default function toWireMessages({ messages }: Params) {
+    return messages.map((message) => {
+        switch (message.role) {
+            case "system":
+                return { role: "system", content: message.content };
+
+            case "user":
+                return {
+                    role: "user",
+                    content:
+                        typeof message.content == "string"
+                            ? message.content
+                            : message.content.map((part) =>
+                                  toWirePart({ part }),
+                              ),
+                };
+
+            case "assistant":
+                return {
+                    role: "assistant",
+                    content: message.content,
+                    ...(message.tool_calls?.length
+                        ? {
+                              tool_calls: message.tool_calls.map((call) => ({
+                                  id: call.id,
+                                  type: "function",
+                                  function: {
+                                      name: call.name,
+                                      arguments: call.arguments,
+                                  },
+                              })),
+                          }
+                        : {}),
+                };
+
+            case "tool":
+                return {
+                    role: "tool",
+                    tool_call_id: message.tool_call_id,
+                    content: message.content,
+                };
+        }
+    });
+}
+
+function toWirePart({ part }: { part: HarnessContentPart }) {
+    switch (part.type) {
+        case "text":
+            return { type: "text", text: part.text };
+
+        case "image":
+            return { type: "image_url", image_url: { url: part.url } };
+
+        case "audio":
+            return {
+                type: "input_audio",
+                input_audio: { data: part.data, format: part.format },
+            };
+    }
+}

src/harness/providers/openai-compatible/transcribe.ts

@@ -0,0 +1,42 @@
+import type { HarnessAdapter } from "../../types";
+import providerRequest from "./provider-request";
+
+/**
+ * `POST /audio/transcriptions` as an OpenAI-style file upload, which
+ * OpenRouter, OpenAI, Groq and local servers all take.
+ */
+const transcribe: HarnessAdapter["transcribe"] = async ({
+    candidate,
+    audio,
+    language,
+    signal,
+}) => {
+    const form = new FormData();
+
+    form.append("model", candidate.model.slug);
+    form.append(
+        "file",
+        new Blob([new Uint8Array(audio.data)], {
+            type: `audio/${audio.format}`,
+        }),
+        `audio.${audio.format}`,
+    );
+
+    if (language) {
+        form.append("language", language);
+    }
+
+    const res = await providerRequest({
+        provider: candidate.provider,
+        path: "/audio/transcriptions",
+        form,
+        signal,
+        candidate_key: candidate.key,
+    });
+
+    const json = await res.json().catch(() => null);
+
+    return { text: String(json?.text || "").trim() };
+};
+
+export default transcribe;

src/harness/providers/openrouter/openrouter-model-to-catalog.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, test } from "bun:test";
+import openRouterModelToCatalog from "./openrouter-model-to-catalog";
+
+describe("openRouterModelToCatalog", () => {
+    test("maps modalities, tools, scores and dates", () => {
+        expect(
+            openRouterModelToCatalog({
+                model: {
+                    id: "nvidia/nemotron-3.5-super-vl:free",
+                    name: "NVIDIA: Nemotron 3.5 Super VL (free)",
+                    created: 1792000000,
+                    context_length: 1000000,
+                    architecture: {
+                        input_modalities: ["text", "image"],
+                        output_modalities: ["text"],
+                    },
+                    supported_parameters: ["tools", "temperature"],
+                    top_provider: { max_completion_tokens: 65536 },
+                    expiration_date: "2026-12-01",
+                    benchmarks: {
+                        artificial_analysis: {
+                            intelligence_index: 26.5,
+                            agentic_index: null,
+                        },
+                    },
+                },
+            }),
+        ).toEqual({
+            slug: "nvidia/nemotron-3.5-super-vl:free",
+            name: "NVIDIA: Nemotron 3.5 Super VL (free)",
+            free: true,
+            input_modalities: ["text", "image"],
+            output_modalities: ["text"],
+            supports_tools: true,
+            context_length: 1000000,
+            max_output_tokens: 65536,
+            voices: undefined,
+            released_at: 1792000000000,
+            expires_at: Date.parse("2026-12-01"),
+            intelligence_score: 26.5,
+            agentic_score: undefined,
+        });
+    });
+
+    test("only :free models and the free router count as free", () => {
+        const free = (id: string) =>
+            openRouterModelToCatalog({ model: { id } }).free;
+
+        expect(free("qwen/qwen3.8-flash:free")).toBe(true);
+        expect(free("openrouter/free")).toBe(true);
+        expect(free("google/lyria-3-pro-preview")).toBe(false);
+    });
+
+    test("fills gaps with text-only defaults", () => {
+        const entry = openRouterModelToCatalog({ model: { id: "a/b" } });
+
+        expect(entry.input_modalities).toEqual(["text"]);
+        expect(entry.supports_tools).toBe(false);
+        expect(entry.expires_at).toBe(undefined);
+    });
+});

src/harness/providers/openrouter/openrouter-model-to-catalog.ts

@@ -0,0 +1,64 @@
+import type { HarnessCatalogModel } from "../../types";
+
+/**
+ * One entry of OpenRouter's `/models` list (the fields the harness reads).
+ */
+export type OpenRouterModel = {
+    id: string;
+    name?: string;
+    created?: number;
+    context_length?: number | null;
+    architecture?: {
+        input_modalities?: string[] | null;
+        output_modalities?: string[] | null;
+    } | null;
+    supported_parameters?: string[] | null;
+    supported_voices?: string[] | null;
+    top_provider?: {
+        max_completion_tokens?: number | null;
+    } | null;
+    expiration_date?: string | null;
+    benchmarks?: {
+        artificial_analysis?: {
+            intelligence_index?: number | null;
+            agentic_index?: number | null;
+        } | null;
+    } | null;
+};
+
+type Params = {
+    model: OpenRouterModel;
+};
+
+/**
+ * Maps an OpenRouter model to a catalog entry. Free means a `:free` model
+ * or the `openrouter/free` router: other models can show zero token
+ * prices and still bill per request, second or image.
+ */
+export default function openRouterModelToCatalog({
+    model,
+}: Params): HarnessCatalogModel {
+    const scores = model.benchmarks?.artificial_analysis;
+    const expires_at = model.expiration_date
+        ? Date.parse(model.expiration_date)
+        : NaN;
+
+    return {
+        slug: model.id,
+        name: model.name || model.id,
+        free: model.id.endsWith(":free") || model.id == "openrouter/free",
+        input_modalities: model.architecture?.input_modalities || ["text"],
+        output_modalities: model.architecture?.output_modalities || ["text"],
+        supports_tools: Boolean(model.supported_parameters?.includes("tools")),
+        context_length: model.context_length || undefined,
+        max_output_tokens:
+            model.top_provider?.max_completion_tokens || undefined,
+        voices: model.supported_voices?.length
+            ? model.supported_voices
+            : undefined,
+        released_at: model.created ? model.created * 1000 : undefined,
+        expires_at: Number.isNaN(expires_at) ? undefined : expires_at,
+        intelligence_score: scores?.intelligence_index ?? undefined,
+        agentic_score: scores?.agentic_index ?? undefined,
+    };
+}

src/harness/providers/parse-retry-at.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, test } from "bun:test";
+import parseRetryAt from "./parse-retry-at";
+
+const now = 1_800_000_000_000;
+const at = (headers: Record<string, string>) =>
+    parseRetryAt({ headers: new Headers(headers), now });
+
+describe("parseRetryAt", () => {
+    test("reads Retry-After seconds and dates", () => {
+        expect(at({ "retry-after": "30" })).toBe(now + 30_000);
+        expect(at({ "retry-after": "Wed, 21 Oct 2026 07:28:00 GMT" })).toBe(
+            Date.parse("Wed, 21 Oct 2026 07:28:00 GMT"),
+        );
+    });
+
+    test("reads reset epochs in ms or seconds", () => {
+        expect(at({ "x-ratelimit-reset": String(now + 5000) })).toBe(
+            now + 5000,
+        );
+        expect(at({ "x-ratelimit-reset": "1800000060" })).toBe(
+            1_800_000_060_000,
+        );
+    });
+
+    test("reads reset durations", () => {
+        expect(at({ "x-ratelimit-reset-requests": "6m0s" })).toBe(
+            now + 360_000,
+        );
+        expect(at({ "x-ratelimit-reset": "1.5s" })).toBe(now + 1500);
+        expect(at({ "x-ratelimit-reset": "20ms" })).toBe(now + 20);
+    });
+
+    test("is undefined without usable headers", () => {
+        expect(at({})).toBe(undefined);
+        expect(at({ "retry-after": "soon" })).toBe(undefined);
+    });
+});

src/harness/providers/parse-retry-at.ts

@@ -0,0 +1,69 @@
+type Params = {
+    headers: Headers;
+    now?: number;
+};
+
+/**
+ * When a rate-limited provider says to come back, as Unix ms. Reads
+ * `Retry-After` (seconds or a date) and `X-RateLimit-Reset` /
+ * `X-RateLimit-Reset-Requests` (an epoch in ms or seconds, a number of
+ * seconds, or a duration such as `6m0s`). Undefined when none is usable.
+ */
+export default function parseRetryAt({
+    headers,
+    now = Date.now(),
+}: Params): number | undefined {
+    const retry_after = headers.get("retry-after");
+
+    if (retry_after) {
+        const seconds = Number(retry_after);
+
+        if (Number.isFinite(seconds)) return now + seconds * 1000;
+
+        const date = Date.parse(retry_after);
+
+        if (!Number.isNaN(date)) return date;
+    }
+
+    for (const name of ["x-ratelimit-reset", "x-ratelimit-reset-requests"]) {
+        const value = headers.get(name);
+
+        if (!value) continue;
+
+        const parsed = parseResetValue({ value, now });
+
+        if (parsed !== undefined) return parsed;
+    }
+
+    return undefined;
+}
+
+function parseResetValue({ value, now }: { value: string; now: number }) {
+    const number = Number(value);
+
+    if (Number.isFinite(number)) {
+        if (number > 1e12) return number;
+        if (number > 1e9) return number * 1000;
+        return now + number * 1000;
+    }
+
+    const parts = [...value.matchAll(/(\d+(?:\.\d+)?)(ms|h|m|s)/g)];
+
+    if (!parts.length) return undefined;
+
+    const unit_ms: Record<string, number> = {
+        ms: 1,
+        s: 1000,
+        m: 60 * 1000,
+        h: 60 * 60 * 1000,
+    };
+
+    return (
+        now +
+        parts.reduce(
+            (total, [, amount, unit]) =>
+                total + Number(amount) * (unit_ms[unit || "s"] || 1000),
+            0,
+        )
+    );
+}

src/harness/providers/to-harness-error.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, test } from "bun:test";
+import toHarnessError from "./to-harness-error";
+
+describe("toHarnessError", () => {
+    test("rate limits are retryable and say when to come back", () => {
+        const error = toHarnessError({
+            status: 429,
+            body: JSON.stringify({
+                error: {
+                    message: "Rate limit exceeded: free-models-per-day",
+                    code: 429,
+                },
+            }),
+            headers: new Headers({ "retry-after": "10" }),
+            provider_name: "OpenRouter",
+        });
+
+        expect(error.message).toBe(
+            "OpenRouter: Rate limit exceeded: free-models-per-day",
+        );
+        expect(error.retryable).toBe(true);
+        expect(error.provider_wide).toBe(false);
+        expect(error.retry_at).toBeGreaterThan(Date.now());
+    });
+
+    test("bad keys and missing credits are provider-wide, not retryable", () => {
+        for (const status of [401, 402]) {
+            const error = toHarnessError({ status, provider_name: "OpenAI" });
+
+            expect(error.provider_wide).toBe(true);
+            expect(error.retryable).toBe(false);
+        }
+    });
+
+    test("a 403 is about the model, not the provider", () => {
+        const error = toHarnessError({
+            status: 403,
+            body: JSON.stringify({
+                error: {
+                    message:
+                        "thinkingmachines/inkling-small:free is only available on agentic harnesses.",
+                },
+            }),
+            provider_name: "OpenRouter",
+        });
+
+        expect(error.provider_wide).toBe(false);
+        expect(error.retryable).toBe(false);
+    });
+
+    test("outages are retryable; bad requests are not", () => {
+        expect(toHarnessError({ status: 503, provider_name: "x" }).retryable).toBe(
+            true,
+        );
+        expect(toHarnessError({ status: 404, provider_name: "x" }).retryable).toBe(
+            false,
+        );
+    });
+
+    test("adds the upstream detail OpenRouter nests in metadata", () => {
+        const error = toHarnessError({
+            status: 400,
+            body: JSON.stringify({
+                error: {
+                    message: "Provider returned error",
+                    metadata: { raw: "tools are not supported" },
+                },
+            }),
+            provider_name: "OpenRouter",
+        });
+
+        expect(error.message).toBe(
+            "OpenRouter: Provider returned error (tools are not supported)",
+        );
+    });
+
+    test("falls back to plain bodies and the status", () => {
+        expect(
+            toHarnessError({ status: 500, body: "boom", provider_name: "x" })
+                .message,
+        ).toBe("x: boom");
+        expect(toHarnessError({ status: 500, provider_name: "x" }).message).toBe(
+            "x: HTTP 500",
+        );
+    });
+});

src/harness/providers/to-harness-error.ts

@@ -0,0 +1,65 @@
+import HarnessError from "../harness-error";
+import parseRetryAt from "./parse-retry-at";
+
+type Params = {
+    status: number;
+    /**
+     * Response body text, usually JSON with an `error`.
+     */
+    body?: string;
+    headers?: Headers;
+    provider_name: string;
+    candidate_key?: string;
+};
+
+/**
+ * Turns a failed provider response into a `HarnessError`:
+ * - 401: bad key; 402: out of credits. Every model on the provider fails
+ *   the same way, so it is `provider_wide`
+ * - 408, 409, 425, 429, 5xx: `retryable`, with `retry_at` when the
+ *   provider says
+ * - anything else (400, 403, 404, 422): only another model can help. A
+ *   403 is about the model (OpenRouter refuses some free models to most
+ *   apps), not the key
+ */
+export default function toHarnessError({
+    status,
+    body,
+    headers,
+    provider_name,
+    candidate_key,
+}: Params) {
+    const detail = readErrorMessage({ body }) || `HTTP ${status}`;
+    const retryable = [408, 409, 425, 429].includes(status) || status >= 500;
+
+    return new HarnessError({
+        message: `${provider_name}: ${detail}`.slice(0, 400),
+        status,
+        retryable,
+        retry_at: retryable && headers ? parseRetryAt({ headers }) : undefined,
+        provider_wide: status == 401 || status == 402,
+        candidate_key,
+    });
+}
+
+function readErrorMessage({ body }: { body?: string }) {
+    if (!body) return undefined;
+
+    try {
+        const json = JSON.parse(body);
+        const error = json?.error ?? json;
+
+        if (typeof error == "string") return error;
+
+        const raw = error?.metadata?.raw;
+        const message = error?.message || json?.message || json?.detail;
+
+        if (typeof message == "string") {
+            return typeof raw == "string" && !message.includes(raw)
+                ? `${message} (${raw.slice(0, 200)})`
+                : message;
+        }
+    } catch {}
+
+    return body.trim().slice(0, 200) || undefined;
+}

src/harness/resolve/create-cooldowns.ts

@@ -0,0 +1,34 @@
+/**
+ * Keys (candidates or providers) that shouldn't be tried until a time.
+ * In memory, per process: a restart forgets them, which only costs one
+ * failed call each.
+ */
+export default function createCooldowns() {
+    const until = new Map<string, number>();
+
+    return {
+        set({ key, at }: { key: string; at: number }) {
+            until.set(key, Math.max(at, until.get(key) || 0));
+        },
+        /**
+         * When the key can be tried again, or undefined if it can now.
+         */
+        get({ key, now = Date.now() }: { key: string; now?: number }) {
+            const at = until.get(key);
+
+            if (at === undefined) return undefined;
+
+            if (at <= now) {
+                until.delete(key);
+                return undefined;
+            }
+
+            return at;
+        },
+        clear() {
+            until.clear();
+        },
+    };
+}
+
+export type HarnessCooldowns = ReturnType<typeof createCooldowns>;

src/harness/resolve/harness-cooldowns.ts

@@ -0,0 +1,9 @@
+import createCooldowns from "./create-cooldowns";
+
+/**
+ * The process-wide cooldowns every harness call shares, so one job's rate
+ * limit spares the next job the same failed call.
+ */
+const HarnessCooldowns = createCooldowns();
+
+export default HarnessCooldowns;

src/harness/resolve/resolve-harness-candidates.ts

@@ -0,0 +1,125 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type {
+    BUN_SQLITE_COBBLE_HARNESS_MODELS,
+    BUN_SQLITE_COBBLE_HARNESS_PROVIDERS,
+} from "@/db/types/db";
+import { Harnesses, type HarnessKind } from "@/src/dict/harness-dict";
+import grabHarnessProviders from "@/src/functions/backend/db/harness/grab-harness-providers";
+import grabHarnessSetting from "@/src/functions/backend/db/harness/grab-harness-setting";
+import harnessModelRowToCatalog from "@/src/functions/backend/db/harness/harness-model-row-to-catalog";
+import resolveProviderConfig from "@/src/functions/backend/db/harness/resolve-provider-config";
+import modelServesHarness from "../catalog/model-serves-harness";
+import rankAutoModels from "../catalog/rank-auto-models";
+import HarnessError from "../harness-error";
+import type {
+    HarnessCandidate,
+    HarnessCatalogModel,
+    HarnessProviderConfig,
+} from "../types";
+
+type Params = {
+    harness: HarnessKind;
+    /**
+     * The bot the call is for: a worker bot can override the default.
+     */
+    bot_id?: number | string;
+};
+
+type CatalogEntry = HarnessCatalogModel & {
+    row_id: number;
+    provider: HarnessProviderConfig;
+};
+
+/**
+ * The models a harness call tries, in order: the pick (the bot's own, else
+ * the super admin's), then
+ * (if it has fallback on, or there is no pick) the automatic free picks.
+ * Only enabled providers with a usable key count, and only the setting's
+ * provider when it has one. Empty when nothing is
+ * set up; throws `unavailable` when the harness is turned off.
+ */
+export default async function resolveHarnessCandidates({
+    harness,
+    bot_id,
+}: Params): Promise<HarnessCandidate[]> {
+    const setting = await grabHarnessSetting({ harness, bot_id });
+
+    if (!setting.enabled) {
+        const title = Harnesses.find((h) => h.value == harness)?.title;
+
+        throw new HarnessError({
+            message:
+                setting.source == "bot"
+                    ? `${title} is turned off for this bot. Turn it on in the bot's model settings.`
+                    : `${title} is turned off. A super admin can turn it on in Settings, under Models.`,
+            code: "unavailable",
+        });
+    }
+
+    const providers = await grabUsableProviders();
+
+    if (!providers.size) return [];
+
+    const rows =
+        (
+            await BunSQLite.sql<BUN_SQLITE_COBBLE_HARNESS_MODELS>({
+                sql: `SELECT * FROM harness_models
+                    WHERE available = 1 AND (free = 1 OR id = ?)`,
+                values: [setting.model_id ?? 0],
+            })
+        ).payload || [];
+
+    const entries: CatalogEntry[] = rows.flatMap((row) => {
+        const provider =
+            row.provider_id && (!setting.provider_id || row.provider_id == setting.provider_id)
+                ? providers.get(row.provider_id)
+                : undefined;
+
+        return provider && row.id
+            ? [{ ...harnessModelRowToCatalog({ row }), row_id: row.id, provider }]
+            : [];
+    });
+
+    const pinned = entries.find(
+        (entry) =>
+            entry.row_id == setting.model_id &&
+            modelServesHarness({ model: entry, harness }),
+    );
+
+    const ordered = [
+        ...(pinned ? [{ entry: pinned, source: "pinned" as const }] : []),
+        ...(!pinned || setting.fallback
+            ? rankAutoModels({ models: entries, harness }).map((entry) => ({
+                  entry,
+                  source: "auto" as const,
+              }))
+            : []),
+    ];
+
+    const seen = new Set<string>();
+
+    return ordered.flatMap(({ entry, source }) => {
+        const key = `${entry.provider.id}:${entry.slug}`;
+
+        if (seen.has(key)) return [];
+        seen.add(key);
+
+        return [{ key, provider: entry.provider, model: entry, source }];
+    });
+}
+
+async function grabUsableProviders() {
+    const rows: BUN_SQLITE_COBBLE_HARNESS_PROVIDERS[] =
+        await grabHarnessProviders({ enabled_only: true });
+    const usable = new Map<number, HarnessProviderConfig>();
+
+    for (const row of rows) {
+        const { config, usable: is_usable } = await resolveProviderConfig({
+            provider: row,
+        });
+
+        if (row.id && is_usable) usable.set(row.id, config);
+    }
+
+    return usable;
+}

src/harness/resolve/with-candidates.test.ts

@@ -0,0 +1,176 @@
+import { describe, expect, test } from "bun:test";
+import HarnessError from "../harness-error";
+import type { HarnessCandidate } from "../types";
+import createCooldowns from "./create-cooldowns";
+import withCandidates from "./with-candidates";
+
+function candidate(slug: string, provider_id = 1): HarnessCandidate {
+    return {
+        key: `${provider_id}:${slug}`,
+        provider: {
+            id: provider_id,
+            kind: "openrouter",
+            base_url: "https://example.test",
+        },
+        model: {
+            slug,
+            input_modalities: ["text"],
+            output_modalities: ["text"],
+            supports_tools: true,
+        },
+        source: "auto",
+    };
+}
+
+const rateLimited = (key: string, retry_at?: number) =>
+    new HarnessError({
+        message: `${key} rate limited`,
+        status: 429,
+        retryable: true,
+        retry_at,
+        candidate_key: key,
+    });
+
+describe("withCandidates", () => {
+    test("falls through to the next model and cools the failed one down", async () => {
+        const cooldowns = createCooldowns();
+        const tried: string[] = [];
+        const a = candidate("a");
+        const b = candidate("b");
+
+        const { result, candidate: used } = await withCandidates({
+            harness: "text",
+            candidates: [a, b],
+            cooldowns,
+            run: async (c) => {
+                tried.push(c.model.slug);
+                if (c == a) throw rateLimited(a.key);
+                return "ok";
+            },
+        });
+
+        expect(result).toBe("ok");
+        expect(used).toBe(b);
+        expect(tried).toEqual(["a", "b"]);
+        expect(cooldowns.get({ key: a.key })).toBeGreaterThan(Date.now());
+
+        tried.length = 0;
+        await withCandidates({
+            harness: "text",
+            candidates: [a, b],
+            cooldowns,
+            run: async (c) => tried.push(c.model.slug),
+        });
+
+        expect(tried).toEqual(["b"]);
+    });
+
+    test("tries the preferred model first", async () => {
+        const tried: string[] = [];
+
+        await withCandidates({
+            harness: "text",
+            candidates: [candidate("a"), candidate("b")],
+            prefer_key: "1:b",
+            cooldowns: createCooldowns(),
+            run: async (c) => tried.push(c.model.slug),
+        });
+
+        expect(tried).toEqual(["b"]);
+    });
+
+    test("a bad key benches every model on that provider", async () => {
+        const cooldowns = createCooldowns();
+        const tried: string[] = [];
+
+        await withCandidates({
+            harness: "text",
+            candidates: [candidate("a", 1), candidate("b", 1), candidate("c", 2)],
+            cooldowns,
+            run: async (c) => {
+                tried.push(c.model.slug);
+                if (c.provider.id == 1) {
+                    throw new HarnessError({
+                        message: "bad key",
+                        status: 401,
+                        provider_wide: true,
+                    });
+                }
+            },
+        });
+
+        expect(tried).toEqual(["a", "c"]);
+    });
+
+    test("reports the last failure, retryable if any failure was", async () => {
+        const promise = withCandidates({
+            harness: "text",
+            candidates: [candidate("a"), candidate("b")],
+            cooldowns: createCooldowns(),
+            run: async (c) => {
+                if (c.model.slug == "a") throw rateLimited("1:a", 5000);
+                throw new HarnessError({ message: "bad request", status: 400 });
+            },
+            now: () => 1000,
+        });
+
+        const error = (await promise.catch((e) => e)) as HarnessError;
+
+        expect(error).toBeInstanceOf(HarnessError);
+        expect(error.message).toBe(
+            "2 text models failed. Last error: bad request",
+        );
+        expect(error.retryable).toBe(true);
+        expect(error.retry_at).toBe(5000);
+    });
+
+    test("is unavailable with nothing to try", async () => {
+        const error = (await withCandidates({
+            harness: "speech",
+            candidates: [],
+            run: async () => {},
+        }).catch((e) => e)) as HarnessError;
+
+        expect(error.code).toBe("unavailable");
+        expect(error.retryable).toBe(false);
+    });
+
+    test("waits instead of calling when everything is cooling down", async () => {
+        const cooldowns = createCooldowns();
+        const a = candidate("a");
+
+        cooldowns.set({ key: a.key, at: Date.now() + 60_000 });
+
+        const error = (await withCandidates({
+            harness: "text",
+            candidates: [a],
+            cooldowns,
+            run: async () => {
+                throw new Error("should not run");
+            },
+        }).catch((e) => e)) as HarnessError;
+
+        expect(error.retryable).toBe(true);
+        expect(error.retry_at).toBeGreaterThan(Date.now());
+    });
+
+    test("cancellation and bugs are not swallowed", async () => {
+        const run = (error: Error) =>
+            withCandidates({
+                harness: "text",
+                candidates: [candidate("a"), candidate("b")],
+                cooldowns: createCooldowns(),
+                run: async () => {
+                    throw error;
+                },
+            }).catch((e) => e);
+
+        const aborted = new HarnessError({ message: "x", code: "aborted" });
+
+        expect(await run(aborted)).toBe(aborted);
+
+        const bug = new TypeError("oops");
+
+        expect(await run(bug)).toBe(bug);
+    });
+});

src/harness/resolve/with-candidates.ts

@@ -0,0 +1,165 @@
+import { AppData } from "@/src/data/app-data";
+import { Harnesses, type HarnessKind } from "@/src/dict/harness-dict";
+import HarnessError from "../harness-error";
+import type { HarnessCandidate } from "../types";
+import type { HarnessCooldowns } from "./create-cooldowns";
+import DefaultCooldowns from "./harness-cooldowns";
+
+type Params<T> = {
+    harness: HarnessKind;
+    candidates: HarnessCandidate[];
+    run: (candidate: HarnessCandidate) => Promise<T>;
+    /**
+     * Tried first when it's a candidate: the model a multi-step run already
+     * uses, so it doesn't switch models mid-task without a reason.
+     */
+    prefer_key?: string;
+    cooldowns?: HarnessCooldowns;
+    now?: () => number;
+};
+
+/**
+ * Bad keys and missing credits keep a provider out this long.
+ */
+const ProviderCooldownMs = 10 * 60 * 1000;
+/**
+ * A model the provider no longer has (404) or won't serve to us (403)
+ * stays out this long.
+ */
+const GoneCooldownMs = 60 * 60 * 1000;
+
+/**
+ * Runs a harness call on the first candidate that works, in order (the
+ * preferred one first), up to `AppData.HarnessMaxCandidates` tries.
+ * Candidates (and providers) on cooldown are skipped; a failure puts its
+ * candidate, or its whole provider for a key problem, on cooldown.
+ *
+ * Throws a `HarnessError`: `unavailable` when there is nothing to try,
+ * otherwise the last failure, `retryable` if any failure was.
+ */
+export default async function withCandidates<T>({
+    harness,
+    candidates,
+    run,
+    prefer_key,
+    cooldowns = DefaultCooldowns,
+    now = Date.now,
+}: Params<T>): Promise<{ result: T; candidate: HarnessCandidate }> {
+    const title = Harnesses.find((h) => h.value == harness)?.title || harness;
+
+    if (!candidates.length) {
+        throw new HarnessError({
+            message: `No ${title.toLowerCase()} model is set up. A super admin can pick one in Settings, under Models.`,
+            code: "unavailable",
+        });
+    }
+
+    const ordered = [
+        ...candidates.filter((c) => c.key == prefer_key),
+        ...candidates.filter((c) => c.key != prefer_key),
+    ];
+
+    const isReady = (candidate: HarnessCandidate) =>
+        !cooldowns.get({ key: candidate.key, now: now() }) &&
+        !cooldowns.get({ key: providerKey(candidate), now: now() });
+
+    if (!ordered.some(isReady)) {
+        const next = Math.min(
+            ...ordered.map((candidate) =>
+                Math.max(
+                    cooldowns.get({ key: candidate.key, now: now() }) || 0,
+                    cooldowns.get({ key: providerKey(candidate), now: now() }) ||
+                        0,
+                ),
+            ),
+        );
+
+        throw new HarnessError({
+            message: `Every ${title.toLowerCase()} model is resting after errors or rate limits. Trying again shortly.`,
+            retryable: true,
+            retry_at: next,
+            resting: true,
+        });
+    }
+
+    const errors: HarnessError[] = [];
+
+    for (const candidate of ordered) {
+        if (errors.length >= AppData.HarnessMaxCandidates) break;
+
+        /**
+         * Checked each time: a failure can bench the rest of its provider.
+         */
+        if (!isReady(candidate)) continue;
+
+        try {
+            return { result: await run(candidate), candidate };
+        } catch (error: unknown) {
+            if (!(error instanceof HarnessError)) throw error;
+            if (error.code == "aborted") throw error;
+
+            errors.push(error);
+            coolDown({ candidate, error, cooldowns, now: now() });
+        }
+    }
+
+    const last = errors[errors.length - 1];
+
+    if (!last) {
+        throw new HarnessError({
+            message: `Every ${title.toLowerCase()} model is resting after errors or rate limits. Trying again shortly.`,
+            retryable: true,
+            resting: true,
+        });
+    }
+
+    const retry_ats = errors
+        .map((error) => error.retry_at)
+        .filter((at): at is number => typeof at == "number");
+
+    throw new HarnessError({
+        message:
+            errors.length > 1
+                ? `${errors.length} ${title.toLowerCase()} models failed. Last error: ${last.message}`
+                : last.message,
+        status: last.status,
+        retryable: errors.some((error) => error.retryable),
+        retry_at: retry_ats.length ? Math.min(...retry_ats) : undefined,
+        candidate_key: last.candidate_key,
+    });
+}
+
+function providerKey(candidate: HarnessCandidate) {
+    return `provider:${candidate.provider.id ?? candidate.provider.base_url}`;
+}
+
+function coolDown({
+    candidate,
+    error,
+    cooldowns,
+    now,
+}: {
+    candidate: HarnessCandidate;
+    error: HarnessError;
+    cooldowns: HarnessCooldowns;
+    now: number;
+}) {
+    if (error.provider_wide) {
+        cooldowns.set({ key: providerKey(candidate), at: now + ProviderCooldownMs });
+        return;
+    }
+
+    if (error.retryable) {
+        const at = error.retry_at || now + AppData.HarnessCooldownMs;
+
+        cooldowns.set({
+            key: candidate.key,
+            at: Math.min(at, now + AppData.HarnessMaxCooldownMs),
+        });
+        return;
+    }
+
+    if (error.status == 403 || error.status == 404) {
+        cooldowns.set({ key: candidate.key, at: now + GoneCooldownMs });
+    }
+}

src/harness/text/fit-transcript-to-context.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessMessage } from "../types";
+import fitTranscriptToContext from "./fit-transcript-to-context";
+
+const big = "x".repeat(30000);
+
+const run = (): HarnessMessage[] => [
+    { role: "user", content: "Do the thing." },
+    { role: "assistant", content: null, tool_calls: [{ id: "1", name: "sandbox_execute_bash", arguments: "{}" }] },
+    { role: "tool", tool_call_id: "1", content: big },
+    { role: "assistant", content: null, tool_calls: [{ id: "2", name: "sandbox_execute_bash", arguments: "{}" }] },
+    { role: "tool", tool_call_id: "2", content: big },
+    { role: "assistant", content: null, tool_calls: [{ id: "3", name: "sandbox_execute_bash", arguments: "{}" }] },
+    { role: "tool", tool_call_id: "3", content: "recent" },
+];
+
+describe("fitTranscriptToContext", () => {
+    test("leaves a transcript that fits alone", () => {
+        const transcript = run();
+        const result = fitTranscriptToContext({ system: "s", transcript, context_length: 200000 });
+
+        expect(result.trimmed).toBe(0);
+        expect(transcript[2]!.content).toBe(big);
+    });
+
+    test("trims the oldest tool output first, keeping the newest", () => {
+        const transcript = run();
+        const result = fitTranscriptToContext({ system: "s", transcript, context_length: 20000 });
+
+        expect(result.trimmed).toBe(1);
+        expect(String(transcript[2]!.content)).toStartWith("[Trimmed: 30000 characters");
+        expect(transcript[4]!.content).toBe(big);
+        expect(transcript[6]!.content).toBe("recent");
+        expect(result.estimated_tokens).toBeLessThanOrEqual(result.budget);
+    });
+
+    test("never trims the newest messages, even over budget", () => {
+        const transcript: HarnessMessage[] = [
+            { role: "user", content: "go" },
+            { role: "tool", tool_call_id: "1", content: big },
+        ];
+        const result = fitTranscriptToContext({ system: "s", transcript, context_length: 1000 });
+
+        expect(result.trimmed).toBe(0);
+        expect(transcript[1]!.content).toBe(big);
+    });
+});

src/harness/text/fit-transcript-to-context.ts

@@ -0,0 +1,92 @@
+import type { HarnessMessage } from "../types";
+
+type Params = {
+    system: string;
+    /**
+     * The conversation after the system prompt. Trimmed in place, so the
+     * run keeps the savings.
+     */
+    transcript: HarnessMessage[];
+    /**
+     * The model's context window, in tokens.
+     */
+    context_length: number;
+    /**
+     * The newest messages, never trimmed: the model needs them to go on.
+     */
+    keep_last?: number;
+};
+
+/**
+ * Share of the window the prompt may fill; the rest is for the answer.
+ */
+const PromptShare = 0.7;
+/**
+ * Rough characters per token, on the low side so the estimate errs high.
+ */
+const CharsPerToken = 3;
+/**
+ * What an image or audio part is counted as, in tokens.
+ */
+const MediaTokens = 1500;
+
+/**
+ * Keeps a long tool run inside the model's context window: while the
+ * estimated prompt is over budget, the oldest tool results (then the
+ * oldest assistant texts) are replaced with a note of what was cut.
+ * Returns how many messages were trimmed.
+ */
+export default function fitTranscriptToContext({
+    system,
+    transcript,
+    context_length,
+    keep_last = 4,
+}: Params) {
+    const budget = Math.floor(context_length * PromptShare);
+    let total = estimateTokens({ text: system }) + transcript.reduce((sum, m) => sum + messageTokens({ message: m }), 0);
+    let trimmed = 0;
+
+    const trimmable = transcript.length - keep_last;
+
+    for (const role of ["tool", "assistant"] as const) {
+        for (let index = 0; index < trimmable && total > budget; index++) {
+            const message = transcript[index]!;
+
+            if (message.role != role || typeof message.content != "string") continue;
+            if (message.content.startsWith("[Trimmed:")) continue;
+
+            const before = messageTokens({ message });
+            const note = `[Trimmed: ${message.content.length} characters of earlier ${role == "tool" ? "tool output" : "notes"} to fit the context.]`;
+
+            transcript[index] = { ...message, content: note } as HarnessMessage;
+            total += messageTokens({ message: transcript[index]! }) - before;
+            trimmed++;
+        }
+    }
+
+    return { trimmed, estimated_tokens: total, budget };
+}
+
+function messageTokens({ message }: { message: HarnessMessage }) {
+    let tokens = 4;
+
+    if (typeof message.content == "string") {
+        tokens += estimateTokens({ text: message.content });
+    } else if (Array.isArray(message.content)) {
+        for (const part of message.content) {
+            tokens += part.type == "text" ? estimateTokens({ text: part.text }) : MediaTokens;
+        }
+    }
+
+    if (message.role == "assistant" && message.tool_calls) {
+        for (const call of message.tool_calls) {
+            tokens += estimateTokens({ text: call.name + call.arguments });
+        }
+    }
+
+    return tokens;
+}
+
+function estimateTokens({ text }: { text: string }) {
+    return Math.ceil(text.length / CharsPerToken);
+}

src/harness/text/generate-text.ts

@@ -0,0 +1,96 @@
+import HarnessError from "../harness-error";
+import grabHarnessAdapter from "../providers/grab-harness-adapter";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import withCandidates from "../resolve/with-candidates";
+import type {
+    HarnessCandidate,
+    HarnessChatResult,
+    HarnessMessage,
+    HarnessToolDefinition,
+} from "../types";
+import prepareMessagesForModel from "./prepare-messages-for-model";
+import stripThinkTags from "./strip-think-tags";
+
+export type GenerateTextParams = {
+    messages: HarnessMessage[];
+    tools?: HarnessToolDefinition[];
+    /**
+     * Resolved for the text harness when not given.
+     */
+    candidates?: HarnessCandidate[];
+    /**
+     * Tried first: the model the run already uses.
+     */
+    prefer_key?: string;
+    max_tokens?: number;
+    temperature?: number;
+    signal?: AbortSignal;
+    /**
+     * Media readings shared across calls of one run.
+     */
+    media_cache?: Map<string, string>;
+    /**
+     * The bot the turn is for: its own models are used where it has them.
+     */
+    bot_id?: number | string;
+};
+
+export type GenerateTextResult = HarnessChatResult & {
+    candidate: HarnessCandidate;
+};
+
+/**
+ * One model turn on the text harness, falling back across its models.
+ * Media the model can't take is read by the helper harnesses first.
+ * Reasoning tags are stripped from the answer.
+ */
+export default async function generateText({
+    messages,
+    tools,
+    candidates,
+    prefer_key,
+    max_tokens,
+    temperature,
+    signal = new AbortController().signal,
+    media_cache = new Map(),
+    bot_id,
+}: GenerateTextParams): Promise<GenerateTextResult> {
+    const { result, candidate } = await withCandidates({
+        harness: "text",
+        candidates:
+            candidates || (await resolveHarnessCandidates({ harness: "text", bot_id })),
+        prefer_key,
+        run: async (candidate) => {
+            const answer = await grabHarnessAdapter({
+                kind: candidate.provider.kind,
+            }).chat({
+                candidate,
+                messages: await prepareMessagesForModel({
+                    messages,
+                    model: candidate.model,
+                    cache: media_cache,
+                    signal,
+                    bot_id,
+                }),
+                tools,
+                max_tokens,
+                temperature,
+                signal,
+            });
+
+            const text = answer.text ? stripThinkTags({ text: answer.text }) : "";
+
+            if (!text && !answer.tool_calls.length) {
+                throw new HarnessError({
+                    message: `The model returned an empty answer.`,
+                    retryable: true,
+                    candidate_key: candidate.key,
+                });
+            }
+
+            return { ...answer, text: text || null };
+        },
+    });
+
+    return { ...result, candidate };
+}

src/harness/text/prepare-messages-for-model.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, test } from "bun:test";
+import { MediaReadNote } from "@/src/data/harness-prompts";
+import type { HarnessMessage } from "../types";
+import prepareMessagesForModel from "./prepare-messages-for-model";
+
+const messages: HarnessMessage[] = [
+    { role: "system", content: "You are a bot." },
+    {
+        role: "user",
+        content: [
+            { type: "text", text: "What's this?" },
+            { type: "image", url: "data:image/png;base64,AAAA", name: "a.png" },
+            { type: "audio", data: "AAAA", format: "mp3" },
+        ],
+    },
+];
+
+const signal = new AbortController().signal;
+
+describe("prepareMessagesForModel", () => {
+    test("keeps media the model can take", async () => {
+        const fitted = await prepareMessagesForModel({
+            messages,
+            model: { input_modalities: ["text", "image", "audio"] },
+            cache: new Map(),
+            signal,
+            read_image: async () => {
+                throw new Error("should not read");
+            },
+        });
+
+        expect(fitted).toEqual(messages);
+    });
+
+    test("reads media the model can't take, once, and notes it", async () => {
+        const cache = new Map<string, string>();
+        let reads = 0;
+        const run = () =>
+            prepareMessagesForModel({
+                messages,
+                model: { input_modalities: ["text"] },
+                cache,
+                signal,
+                read_image: async () => {
+                    reads++;
+                    return "A red square.";
+                },
+                transcribe: async () => "hello there",
+            });
+
+        const fitted = await run();
+        await run();
+
+        expect(reads).toBe(1);
+        expect(fitted[0]).toEqual({
+            role: "system",
+            content: `You are a bot.\n\n${MediaReadNote}`,
+        });
+        expect(fitted[1]).toEqual({
+            role: "user",
+            content: [
+                { type: "text", text: "What's this?" },
+                { type: "text", text: `[Image "a.png": A red square.]` },
+                { type: "text", text: "[Audio, transcribed: hello there]" },
+            ],
+        });
+    });
+
+    test("says when a part can't be read instead of failing", async () => {
+        const fitted = await prepareMessagesForModel({
+            messages: [messages[1]!],
+            model: { input_modalities: ["text", "audio"] },
+            cache: new Map(),
+            signal,
+            read_image: async () => {
+                throw new Error("No media reading model is set up.");
+            },
+        });
+
+        expect(fitted[0]).toMatchObject({
+            content: [
+                {},
+                {
+                    type: "text",
+                    text: `[Image "a.png": couldn't be read. No media reading model is set up.]`,
+                },
+                { type: "audio" },
+            ],
+        });
+    });
+});

src/harness/text/prepare-messages-for-model.ts

@@ -0,0 +1,148 @@
+import { MediaReadNote } from "@/src/data/harness-prompts";
+import transcribeAudio from "../audio/transcribe-audio";
+import HarnessError from "../harness-error";
+import readImage from "../media/read-image";
+import type {
+    HarnessContentPart,
+    HarnessMessage,
+    HarnessModelInfo,
+} from "../types";
+
+type Params = {
+    messages: HarnessMessage[];
+    model: Pick<HarnessModelInfo, "input_modalities">;
+    /**
+     * Readings by media key, kept across candidates and steps so nothing is
+     * read twice.
+     */
+    cache: Map<string, string>;
+    signal: AbortSignal;
+    /**
+     * Reads media with this bot's models when it has its own.
+     */
+    bot_id?: number | string;
+    read_image?: typeof readImage;
+    transcribe?: typeof transcribeAudio;
+};
+
+/**
+ * Fits messages to a model's inputs: images it can't see are read by the
+ * media reading harness and audio it can't hear is transcribed, each
+ * replaced with a `[bracketed]` reading. A part that can't be read says so
+ * instead of failing the call.
+ */
+export default async function prepareMessagesForModel({
+    messages,
+    model,
+    cache,
+    signal,
+    bot_id,
+    read_image = readImage,
+    transcribe = transcribeAudio,
+}: Params): Promise<HarnessMessage[]> {
+    const sees = model.input_modalities.includes("image");
+    const hears = model.input_modalities.includes("audio");
+    let replaced = false;
+
+    const fitPart = async (
+        part: HarnessContentPart,
+    ): Promise<HarnessContentPart> => {
+        if (part.type == "image" && !sees) {
+            replaced = true;
+            return {
+                type: "text",
+                text: await readOnce({
+                    key: `image:${Bun.hash(part.url)}`,
+                    label: `Image${part.name ? ` "${part.name}"` : ""}`,
+                    cache,
+                    signal,
+                    read: () => read_image({ url: part.url, signal, bot_id }),
+                }),
+            };
+        }
+
+        if (part.type == "audio" && !hears) {
+            replaced = true;
+            return {
+                type: "text",
+                text: await readOnce({
+                    key: `audio:${Bun.hash(part.data)}`,
+                    label: `Audio${part.name ? ` "${part.name}"` : ""}, transcribed`,
+                    cache,
+                    signal,
+                    read: () =>
+                        transcribe({
+                            audio: {
+                                data: Buffer.from(part.data, "base64"),
+                                format: part.format,
+                            },
+                            signal,
+                            bot_id,
+                        }),
+                }),
+            };
+        }
+
+        return part;
+    };
+
+    const fitted: HarnessMessage[] = [];
+
+    for (const message of messages) {
+        if (message.role == "user" && typeof message.content != "string") {
+            const content: HarnessContentPart[] = [];
+
+            for (const part of message.content) {
+                content.push(await fitPart(part));
+            }
+
+            fitted.push({ ...message, content });
+        } else {
+            fitted.push(message);
+        }
+    }
+
+    if (!replaced) return fitted;
+
+    const system = fitted.findIndex((message) => message.role == "system");
+
+    if (system >= 0) {
+        const message = fitted[system] as { role: "system"; content: string };
+        fitted[system] = {
+            role: "system",
+            content: `${message.content}\n\n${MediaReadNote}`,
+        };
+    }
+
+    return fitted;
+}
+
+async function readOnce({
+    key,
+    label,
+    cache,
+    signal,
+    read,
+}: {
+    key: string;
+    label: string;
+    cache: Map<string, string>;
+    signal: AbortSignal;
+    read: () => Promise<string>;
+}) {
+    const cached = cache.get(key);
+
+    if (cached !== undefined) return cached;
+
+    try {
+        const text = `[${label}: ${await read()}]`;
+        cache.set(key, text);
+        return text;
+    } catch (error: unknown) {
+        if (signal.aborted || (error instanceof HarnessError && error.code == "aborted")) {
+            throw error;
+        }
+
+        return `[${label}: couldn't be read. ${error instanceof Error ? error.message : String(error)}]`;
+    }
+}

src/harness/text/run-text-loop.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, test } from "bun:test";
+import { z } from "zod";
+import HarnessError from "../harness-error";
+import type { HarnessCandidate, HarnessTool } from "../types";
+import type { GenerateTextParams, GenerateTextResult } from "./generate-text";
+import runTextLoop from "./run-text-loop";
+
+const candidate: HarnessCandidate = {
+    key: "1:m",
+    provider: { id: 1, kind: "openrouter", base_url: "https://x.test" },
+    model: {
+        slug: "m",
+        input_modalities: ["text"],
+        output_modalities: ["text"],
+        supports_tools: true,
+    },
+    source: "auto",
+};
+
+const memory: string[] = [];
+
+const remember: HarnessTool<{ note: string }> = {
+    name: "remember",
+    description: "Keeps a note.",
+    schema: z.object({ note: z.string() }),
+    run: async ({ args }) => {
+        memory.push(args.note);
+        return "Saved.";
+    },
+};
+
+/**
+ * A fake model that plays back scripted turns and records what it saw.
+ */
+function scripted(turns: Partial<GenerateTextResult>[]) {
+    const seen: GenerateTextParams[] = [];
+
+    const generate = async (params: GenerateTextParams) => {
+        seen.push(structuredClone({ ...params, signal: undefined }));
+        const turn = turns[seen.length - 1] || { text: "Done." };
+
+        return {
+            text: null,
+            tool_calls: [],
+            candidate,
+            ...turn,
+        } as GenerateTextResult;
+    };
+
+    return { generate, seen };
+}
+
+const resolveCandidates = async () => [candidate];
+
+describe("runTextLoop", () => {
+    test("runs tools and feeds their results back until the model answers", async () => {
+        memory.length = 0;
+        const { generate, seen } = scripted([
+            {
+                text: "Noting that.",
+                tool_calls: [
+                    { id: "c1", name: "remember", arguments: `{"note":"tea"}` },
+                ],
+                usage: { prompt_tokens: 10, completion_tokens: 3, cost: 0.5 },
+            },
+            {
+                text: "I'll remember you like tea.",
+                usage: { prompt_tokens: 20, completion_tokens: 5 },
+            },
+        ]);
+        const steps: number[] = [];
+
+        const result = await runTextLoop({
+            system: "You are a bot.",
+            messages: [{ role: "user", content: "I like tea." }],
+            tools: [remember],
+            generate,
+            resolveCandidates,
+            onStep: ({ index }) => {
+                steps.push(index);
+            },
+        });
+
+        expect(result.text).toBe("I'll remember you like tea.");
+        expect(result.steps).toBe(2);
+        expect(steps).toEqual([0, 1]);
+        expect(memory).toEqual(["tea"]);
+        expect(result.usage).toEqual({
+            prompt_tokens: 30,
+            completion_tokens: 8,
+            cost: 0.5,
+        });
+        expect(seen[0]!.messages[0]).toEqual({
+            role: "system",
+            content: "You are a bot.",
+        });
+        expect(seen[0]!.tools?.[0]?.name).toBe("remember");
+        expect(seen[1]!.prefer_key).toBe("1:m");
+        expect(seen[1]!.messages.slice(-2)).toEqual([
+            {
+                role: "assistant",
+                content: "Noting that.",
+                tool_calls: [
+                    { id: "c1", name: "remember", arguments: `{"note":"tea"}` },
+                ],
+            },
+            { role: "tool", tool_call_id: "c1", content: "Saved." },
+        ]);
+    });
+
+    test("asks the model to wrap up, without tools, after the last step", async () => {
+        const loop_call = {
+            tool_calls: [{ id: "c", name: "remember", arguments: `{"note":"x"}` }],
+        };
+        const { generate, seen } = scripted([
+            loop_call,
+            loop_call,
+            { text: "Ran out of room; here's where I got to.", ...loop_call },
+        ]);
+
+        const result = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "go" }],
+            tools: [remember],
+            max_steps: 2,
+            generate,
+            resolveCandidates,
+        });
+
+        expect(seen).toHaveLength(3);
+        expect(seen[2]!.tools).toBe(undefined);
+        expect(seen[2]!.messages.at(-1)).toMatchObject({ role: "user" });
+        expect(result.text).toBe("Ran out of room; here's where I got to.");
+        expect(result.messages.at(-1)).toEqual({
+            role: "assistant",
+            content: "Ran out of room; here's where I got to.",
+        });
+    });
+
+    test("stops when cancelled", async () => {
+        const controller = new AbortController();
+        controller.abort();
+
+        const error = await runTextLoop({
+            system: "s",
+            messages: [{ role: "user", content: "go" }],
+            signal: controller.signal,
+            generate: scripted([]).generate,
+            resolveCandidates,
+        }).catch((e) => e);
+
+        expect(error).toBeInstanceOf(HarnessError);
+        expect(error.code).toBe("aborted");
+    });
+});

src/harness/text/run-text-loop.ts

@@ -0,0 +1,167 @@
+import { AppData } from "@/src/data/app-data";
+import HarnessError from "../harness-error";
+import resolveHarnessCandidates from "../resolve/resolve-harness-candidates";
+import runToolCall from "../tools/run-tool-call";
+import toolDefinition from "../tools/tool-definition";
+import type {
+    HarnessCandidate,
+    HarnessMessage,
+    HarnessTool,
+    HarnessToolCall,
+    HarnessUsage,
+} from "../types";
+import fitTranscriptToContext from "./fit-transcript-to-context";
+import generateText from "./generate-text";
+
+export type TextLoopStep = {
+    index: number;
+    text: string | null;
+    tool_calls: HarnessToolCall[];
+    candidate: HarnessCandidate;
+};
+
+type Params = {
+    /**
+     * The bot's full prompt (`grabBotPrompt`).
+     */
+    system: string;
+    /**
+     * The conversation so far, ending with what to do now.
+     */
+    messages: HarnessMessage[];
+    tools?: HarnessTool[];
+    max_steps?: number;
+    signal?: AbortSignal;
+    /**
+     * Called after each model turn, before its tools run. Use it for
+     * progress and checkpoints.
+     */
+    onStep?: (step: TextLoopStep) => Promise<void> | void;
+    /**
+     * The bot the run is for: its own models are used where it has them.
+     */
+    bot_id?: number | string;
+    generate?: typeof generateText;
+    resolveCandidates?: () => Promise<HarnessCandidate[]>;
+};
+
+export type TextLoopResult = {
+    /**
+     * The final answer for the user.
+     */
+    text: string;
+    /**
+     * Everything after the system prompt, including tool turns.
+     */
+    messages: HarnessMessage[];
+    steps: number;
+    candidate: HarnessCandidate;
+    usage: HarnessUsage;
+};
+
+const WrapUpPrompt = `You've used every step this run allows. Don't call any more tools. Report what you did, what is left, and anything you need from the user.`;
+
+/**
+ * The text harness at work: the model answers or calls tools, tool results
+ * go back to it, and so on until it answers without calling a tool. Tools
+ * run one at a time, in order. Stays on one model unless it fails. After
+ * `max_steps` turns the model is asked to wrap up without tools.
+ */
+export default async function runTextLoop({
+    system,
+    messages,
+    tools = [],
+    max_steps = AppData.HarnessMaxToolSteps,
+    signal = new AbortController().signal,
+    onStep,
+    bot_id,
+    generate = generateText,
+    resolveCandidates = () => resolveHarnessCandidates({ harness: "text", bot_id }),
+}: Params): Promise<TextLoopResult> {
+    const candidates = await resolveCandidates();
+    const definitions = tools.map((tool) => toolDefinition({ tool }));
+    const media_cache = new Map<string, string>();
+    const transcript: HarnessMessage[] = [...messages];
+    const usage: Required<HarnessUsage> = {
+        prompt_tokens: 0,
+        completion_tokens: 0,
+        cost: 0,
+    };
+    let prefer_key: string | undefined;
+
+    for (let index = 0; index <= max_steps; index++) {
+        if (signal.aborted) {
+            throw new HarnessError({ message: `Cancelled.`, code: "aborted" });
+        }
+
+        const wrap_up = index == max_steps;
+
+        if (wrap_up) {
+            transcript.push({ role: "user", content: WrapUpPrompt });
+        }
+
+        /**
+         * Sized for the model the run is on (or will start on). A fallback
+         * with a smaller window gets a 400 and the next model is tried.
+         */
+        const model = (candidates.find((c) => c.key == prefer_key) || candidates[0])?.model;
+
+        fitTranscriptToContext({
+            system,
+            transcript,
+            context_length: model?.context_length || AppData.HarnessAutoMinContext,
+        });
+
+        const answer = await generate({
+            messages: [{ role: "system", content: system }, ...transcript],
+            tools: wrap_up || !definitions.length ? undefined : definitions,
+            candidates,
+            prefer_key,
+            signal,
+            media_cache,
+            bot_id,
+        });
+
+        prefer_key = answer.candidate.key;
+        usage.prompt_tokens += answer.usage?.prompt_tokens || 0;
+        usage.completion_tokens += answer.usage?.completion_tokens || 0;
+        usage.cost += answer.usage?.cost || 0;
+
+        const tool_calls = wrap_up ? [] : answer.tool_calls;
+
+        transcript.push({
+            role: "assistant",
+            content: answer.text,
+            ...(tool_calls.length ? { tool_calls } : {}),
+        });
+
+        await onStep?.({
+            index,
+            text: answer.text,
+            tool_calls,
+            candidate: answer.candidate,
+        });
+
+        if (!tool_calls.length) {
+            return {
+                text:
+                    answer.text ||
+                    (wrap_up ? `I ran out of steps before I could finish.` : ""),
+                messages: transcript,
+                steps: index + 1,
+                candidate: answer.candidate,
+                usage,
+            };
+        }
+
+        for (const call of tool_calls) {
+            transcript.push({
+                role: "tool",
+                tool_call_id: call.id,
+                content: await runToolCall({ call, tools, signal }),
+            });
+        }
+    }
+
+    throw new Error(`Unreachable: the text loop ended without an answer.`);
+}

src/harness/text/strip-think-tags.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, test } from "bun:test";
+import stripThinkTags from "./strip-think-tags";
+
+describe("stripThinkTags", () => {
+    test("removes reasoning blocks", () => {
+        expect(
+            stripThinkTags({ text: "<think>plan\nmore</think>\n\nDone." }),
+        ).toBe("Done.");
+        expect(
+            stripThinkTags({ text: "A <THINK>x</THINK>B <think>y</think>C" }),
+        ).toBe("A B C");
+    });
+
+    test("drops an unclosed block at the start", () => {
+        expect(stripThinkTags({ text: "<think>never ends" })).toBe("");
+    });
+
+    test("leaves plain text alone", () => {
+        expect(stripThinkTags({ text: " Hello " })).toBe("Hello");
+    });
+});

src/harness/text/strip-think-tags.ts

@@ -0,0 +1,14 @@
+type Params = {
+    text: string;
+};
+
+/**
+ * Removes the `<think>...</think>` reasoning some models write into their
+ * answer, including an unclosed one at the start.
+ */
+export default function stripThinkTags({ text }: Params) {
+    return text
+        .replace(/<think>[\s\S]*?<\/think>/gi, "")
+        .replace(/^\s*<think>[\s\S]*$/i, "")
+        .trim();
+}

src/harness/tools/bot/delegate-task-tool.ts

@@ -0,0 +1,77 @@
+import { z } from "zod";
+import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
+import pushBotNotice from "@/src/bots-queue/push-bot-notice";
+import enqueueBotJob from "@/src/functions/backend/db/bots-queue/enqueue-bot-job";
+import insertBotMessage from "@/src/functions/backend/db/bots/insert-bot-message";
+import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers";
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    user_id: number | string;
+    /**
+     * The coordinator's name, shown in the worker's conversation.
+     */
+    coordinator_name: string;
+    /**
+     * The user's worker bots.
+     */
+    workers: BUN_SQLITE_COBBLE_BOTS[];
+};
+
+type Args = {
+    bot_id: number;
+    title: string;
+    instructions: string;
+};
+
+/**
+ * Lets the coordinator hand a task to a worker bot. The worker runs it on
+ * its own main computer (if it has one) and posts the result in its own
+ * conversation, where a note says who asked.
+ */
+export default function delegateTaskTool({
+    user_id,
+    coordinator_name,
+    workers,
+}: Params): HarnessTool<Args> {
+    const crew = workers
+        .map((bot) => `${bot.id}: ${bot.name}${bot.role ? ` (${bot.role})` : ""}`)
+        .join("; ");
+
+    return {
+        name: "delegate_task",
+        description: `Hand a task to a crew member. They run it in the background and post the result in their own conversation. Write the instructions as a full brief: what to do, where, and how they'll know it's done. Crew: ${crew}.`,
+        schema: z.object({
+            bot_id: z.number().int(),
+            title: z.string().min(1).max(80),
+            instructions: z.string().min(1).max(8000),
+        }),
+        run: async ({ args }) => {
+            const worker = workers.find((bot) => bot.id == args.bot_id);
+
+            if (!worker?.id) {
+                throw new Error(`No crew member has id ${args.bot_id}. Crew: ${crew}.`);
+            }
+
+            const computers = await grabBotComputers({ bot_id: worker.id, user_id });
+
+            const job_id = await enqueueBotJob({
+                user_id,
+                bot_id: worker.id,
+                title: args.title,
+                instructions: args.instructions,
+                computer_id: computers[0]?.id || undefined,
+            });
+
+            await pushBotNotice({
+                bot_id: worker.id,
+                message: await insertBotMessage({
+                    bot_id: worker.id,
+                    text: `${coordinator_name} asked me to: **${args.title}**`,
+                }),
+            });
+
+            return `Task #${job_id} is with ${worker.name}. Their result will be in their conversation.`;
+        },
+    };
+}

src/harness/tools/bot/grab-bot-chat-tools.ts

@@ -0,0 +1,60 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
+import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers";
+import type { HarnessTool } from "../../types";
+import delegateTaskTool from "./delegate-task-tool";
+import grabBotTools from "./grab-bot-tools";
+import startTaskTool from "./start-task-tool";
+
+type Params = {
+    bot: BUN_SQLITE_COBBLE_BOTS;
+    user_id: number | string;
+    /**
+     * The user message being answered.
+     */
+    message_id?: number | string;
+};
+
+/**
+ * The tools a bot replies in chat with: its own (memory, instructions),
+ * start_task for longer work, and for the coordinator delegate_task to
+ * the crew. Computers are reached through tasks, never mid-reply.
+ */
+export default async function grabBotChatTools({
+    bot,
+    user_id,
+    message_id,
+}: Params): Promise<HarnessTool[]> {
+    const bot_id = bot.id || 0;
+    const tools = [
+        ...grabBotTools({ user_id, bot_id }),
+        startTaskTool({
+            user_id,
+            bot_id,
+            computers: await grabBotComputers({ bot_id, user_id }),
+            message_id,
+        }),
+    ];
+
+    if (bot.kind != "coordinator") return tools;
+
+    const workers =
+        (
+            await BunSQLite.sql<BUN_SQLITE_COBBLE_BOTS>({
+                sql: `SELECT * FROM bots WHERE user_id = ? AND kind = 'worker' ORDER BY id ASC`,
+                values: [user_id],
+            })
+        ).payload || [];
+
+    if (workers.length) {
+        tools.push(
+            delegateTaskTool({
+                user_id,
+                coordinator_name: bot.name || "The coordinator",
+                workers,
+            }),
+        );
+    }
+
+    return tools;
+}

src/harness/tools/bot/grab-bot-tools.ts

@@ -0,0 +1,19 @@
+import type { HarnessTool } from "../../types";
+import updateInstructionsTool from "./update-instructions-tool";
+import updateMemoryTool from "./update-memory-tool";
+
+type Params = {
+    user_id: number | string;
+    bot_id: number | string;
+};
+
+/**
+ * The tools a bot works with, bound to that bot. Computer, delegation and
+ * media tools join this list as they are built.
+ */
+export default function grabBotTools({ user_id, bot_id }: Params): HarnessTool[] {
+    return [
+        updateMemoryTool({ user_id, bot_id }),
+        updateInstructionsTool({ user_id, bot_id }),
+    ];
+}

src/harness/tools/bot/start-task-tool.ts

@@ -0,0 +1,66 @@
+import { z } from "zod";
+import type { BUN_SQLITE_COBBLE_COMPUTERS } from "@/db/types/db";
+import enqueueBotJob from "@/src/functions/backend/db/bots-queue/enqueue-bot-job";
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    user_id: number | string;
+    bot_id: number | string;
+    /**
+     * The bot's computers, its main one first (`grabBotComputers`).
+     */
+    computers: BUN_SQLITE_COBBLE_COMPUTERS[];
+    /**
+     * The user message that asked for the work.
+     */
+    message_id?: number | string;
+};
+
+type Args = {
+    title: string;
+    instructions: string;
+    computer_id?: number;
+};
+
+/**
+ * Lets a bot queue long-running work for itself. The task runs in the
+ * background with the computer's tools and posts its result to the chat.
+ */
+export default function startTaskTool({
+    user_id,
+    bot_id,
+    computers,
+    message_id,
+}: Params): HarnessTool<Args> {
+    const listed = computers
+        .map((computer) => `${computer.id} (${computer.name || computer.kind})`)
+        .join(", ");
+
+    return {
+        name: "start_task",
+        description: `Queue work that takes a while or needs a computer. It runs in the background and its result is posted in this chat. Write the instructions as a full brief for yourself: what to do, where, and how you'll know it's done. ${
+            computers.length
+                ? `Computers you can use: ${listed}. Defaults to ${computers[0]!.id}.`
+                : `You have no computer, so the task can't run commands or browse.`
+        }`,
+        schema: z.object({
+            title: z.string().min(1).max(80).describe("A short label, e.g. \"Check the backup logs\""),
+            instructions: z.string().min(1).max(8000),
+            computer_id: z.number().int().optional(),
+        }),
+        run: async ({ args }) => {
+            const computer_id = args.computer_id ?? computers[0]?.id ?? undefined;
+
+            const job_id = await enqueueBotJob({
+                user_id,
+                bot_id,
+                title: args.title,
+                instructions: args.instructions,
+                message_id,
+                computer_id: computer_id || undefined,
+            });
+
+            return `Task #${job_id} queued. Tell the user you're on it; the result will be posted here when it's done.`;
+        },
+    };
+}

src/harness/tools/bot/update-instructions-tool.ts

@@ -0,0 +1,33 @@
+import { z } from "zod";
+import { AppData } from "@/src/data/app-data";
+import updateBotSystemPrompt from "@/src/functions/backend/db/bots/update-bot-system-prompt";
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    user_id: number | string;
+    bot_id: number | string;
+};
+
+/**
+ * Lets a bot rewrite its own core instructions as it adapts to the user.
+ */
+export default function updateInstructionsTool({
+    user_id,
+    bot_id,
+}: Params): HarnessTool<{ instructions: string }> {
+    return {
+        name: "update_instructions",
+        description: `Replace your core instructions with this full text (markdown): the start of your prompt, before any workspace or memory. Change them only to work better for this user, such as a new standing rule they gave you. Keep everything that still applies.`,
+        schema: z.object({
+            instructions: z.string().trim().min(1).max(AppData.PromptMaxLength),
+        }),
+        run: async ({ args }) => {
+            await updateBotSystemPrompt({
+                user_id,
+                bot_id,
+                system_prompt: args.instructions,
+            });
+            return `Instructions saved. They apply from your next run.`;
+        },
+    };
+}

src/harness/tools/bot/update-memory-tool.ts

@@ -0,0 +1,29 @@
+import { z } from "zod";
+import { AppData } from "@/src/data/app-data";
+import updateBotMemory from "@/src/functions/backend/db/bots/update-bot-memory";
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    user_id: number | string;
+    bot_id: number | string;
+};
+
+/**
+ * Lets a bot rewrite its own memory (shown to it under `# Memory`).
+ */
+export default function updateMemoryTool({
+    user_id,
+    bot_id,
+}: Params): HarnessTool<{ memory: string }> {
+    return {
+        name: "update_memory",
+        description: `Replace your memory with this full text (markdown). It's in your prompt under "# Memory" on every run. Keep what you must always remember: the user's preferences, facts about their work, decisions made. Anything you leave out is forgotten, so include the parts that still matter.`,
+        schema: z.object({
+            memory: z.string().max(AppData.PromptMaxLength),
+        }),
+        run: async ({ args }) => {
+            await updateBotMemory({ user_id, bot_id, memory: args.memory });
+            return `Memory saved.`;
+        },
+    };
+}

src/harness/tools/computer/compose-computer-env-files.ts

@@ -0,0 +1,18 @@
+type Params = {
+    envs: { key: string; value: string }[];
+};
+
+/**
+ * A computer's env files: `shell` for bash (`export KEY='value'`, single
+ * quotes escaped) and `json` for code runners.
+ */
+export default function composeComputerEnvFiles({ envs }: Params) {
+    const shell = envs
+        .map((env) => `export ${env.key}='${env.value.replaceAll("'", `'\\''`)}'`)
+        .join("\n");
+
+    return {
+        shell: shell ? `${shell}\n` : "",
+        json: JSON.stringify(Object.fromEntries(envs.map((env) => [env.key, env.value]))),
+    };
+}

src/harness/tools/computer/open-computer-tools.ts

@@ -0,0 +1,88 @@
+import { ComputerMcpExcludedTools } from "@/src/dict/computer-dict";
+import grabComputerMcpUrl from "@/src/functions/backend/db/computers/grab-computer-mcp-url";
+import grabComputerEnvs from "@/src/functions/backend/db/envs/grab-computer-envs";
+import redactSecrets from "@/src/utils/redact-secrets";
+import connectMcpServer from "../../mcp/connect-mcp-server";
+import grabMcpTools from "../../mcp/grab-mcp-tools";
+import readImage from "../../media/read-image";
+import type { HarnessTool } from "../../types";
+import withComputerEnv from "./with-computer-env";
+import writeComputerEnv from "./write-computer-env";
+
+type Params = {
+    computer_id: number | string;
+    /**
+     * Screenshots are read with this bot's media reading model, and only
+     * workspaces it shares with the computer add envs.
+     */
+    bot_id?: number | string;
+    signal?: AbortSignal;
+};
+
+export type OpenComputer = {
+    name: string;
+    tools: HarnessTool[];
+    /**
+     * The envs set on the computer, for the prompt: keys and descriptions,
+     * never values.
+     */
+    envs: { key: string; description: string }[];
+    /**
+     * Blanks the computer's secret values out of text, e.g. the report.
+     */
+    redact: (text: string) => string;
+    close: () => Promise<void>;
+};
+
+/**
+ * Connects to a computer (its AIO sandbox, over MCP) and returns its tools:
+ * shell, files, code and browser. Its envs (`grabComputerEnvs`) are written
+ * to the computer and loaded by the shell and code tools; secret values are
+ * redacted from everything the tools return. Screenshots come back as
+ * readings from the media reading harness. Call `close` when the run ends.
+ * Throws when the computer can't be reached, which the queue retries.
+ */
+export default async function openComputerTools({
+    computer_id,
+    bot_id,
+    signal,
+}: Params): Promise<OpenComputer> {
+    const target = await grabComputerMcpUrl({ computer_id });
+
+    if (!target) {
+        throw new Error(`Computer #${computer_id} has no container to connect to.`);
+    }
+
+    const envs = await grabComputerEnvs({ computer_id, bot_id });
+    const paths = await writeComputerEnv({ base_url: target.base_url, envs, signal });
+    const secrets = envs.filter((env) => env.secret);
+    const redact = (text: string) => redactSecrets({ text, secrets });
+
+    const client = await connectMcpServer({ url: target.url, signal });
+
+    try {
+        const tools = await grabMcpTools({
+            client,
+            exclude: ComputerMcpExcludedTools,
+            redact,
+            describeImage: ({ data, mime_type }) =>
+                readImage({
+                    url: `data:${mime_type};base64,${data}`,
+                    focus: "This is a screenshot of a computer screen. Describe the page layout, text, buttons and form fields, and any errors or popups.",
+                    signal,
+                    bot_id,
+                }),
+        });
+
+        return {
+            name: target.name,
+            tools: tools.map((tool) => withComputerEnv({ tool, ...paths })),
+            envs: envs.map((env) => ({ key: env.key, description: env.description })),
+            redact,
+            close: () => client.close().catch(() => {}),
+        };
+    } catch (error) {
+        await client.close().catch(() => {});
+        throw error;
+    }
+}

src/harness/tools/computer/with-computer-env.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, test } from "bun:test";
+import type { HarnessTool } from "../../types";
+import composeComputerEnvFiles from "./compose-computer-env-files";
+import withComputerEnv from "./with-computer-env";
+
+const echoTool = (name: string): HarnessTool => ({
+    name,
+    description: "",
+    parameters: {},
+    run: async ({ args }) => JSON.stringify(args),
+});
+
+const paths = { shell_path: "/home/gem/.cobble/env", json_path: "/home/gem/.cobble/env.json" };
+const signal = new AbortController().signal;
+
+describe("withComputerEnv", () => {
+    test("sources the env file before bash commands", async () => {
+        const tool = withComputerEnv({ tool: echoTool("sandbox_execute_bash"), ...paths });
+        const sent = JSON.parse(await tool.run({ args: { cmd: "echo $TOKEN" }, signal }));
+
+        expect(sent.cmd).toBe("[ -f /home/gem/.cobble/env ] && . /home/gem/.cobble/env; echo $TOKEN");
+    });
+
+    test("loads the json file before python code", async () => {
+        const tool = withComputerEnv({ tool: echoTool("sandbox_execute_code"), ...paths });
+        const sent = JSON.parse(await tool.run({ args: { code: "print(1)", language: "python" }, signal }));
+
+        expect(sent.code).toStartWith("import json as _cobble_json");
+        expect(sent.code).toEndWith("\nprint(1)");
+    });
+
+    test("leaves other tools alone", () => {
+        const tool = echoTool("browser_navigate");
+        expect(withComputerEnv({ tool, ...paths })).toBe(tool);
+    });
+});
+
+describe("composeComputerEnvFiles", () => {
+    test("escapes single quotes for the shell", () => {
+        expect(
+            composeComputerEnvFiles({ envs: [{ key: "A", value: "it's" }] }).shell,
+        ).toBe(`export A='it'\\''s'\n`);
+    });
+
+    test("is empty without envs", () => {
+        expect(composeComputerEnvFiles({ envs: [] })).toEqual({ shell: "", json: "{}" });
+    });
+});

src/harness/tools/computer/with-computer-env.ts

@@ -0,0 +1,66 @@
+import type { HarnessTool } from "../../types";
+
+type Params = {
+    tool: HarnessTool;
+    /**
+     * Absolute paths of the env files on the computer.
+     */
+    shell_path: string;
+    json_path: string;
+};
+
+/**
+ * Makes a computer tool run with the computer's envs. The sandbox's shells
+ * don't read a profile, so bash commands source the env file first and
+ * Python or JavaScript code loads the JSON one. The model never sees the
+ * addition; other tools pass through.
+ */
+export default function withComputerEnv({
+    tool,
+    shell_path,
+    json_path,
+}: Params): HarnessTool {
+    const shell_prefix = `[ -f ${shell_path} ] && . ${shell_path}; `;
+
+    if (tool.name == "sandbox_execute_bash") {
+        return {
+            ...tool,
+            run: ({ args, signal }) =>
+                tool.run({
+                    args: typeof args?.cmd == "string" ? { ...args, cmd: `${shell_prefix}${args.cmd}` } : args,
+                    signal,
+                }),
+        };
+    }
+
+    if (tool.name == "sandbox_execute_code") {
+        return {
+            ...tool,
+            run: ({ args, signal }) =>
+                tool.run({
+                    args:
+                        typeof args?.code == "string"
+                            ? { ...args, code: `${codePrefix({ language: args.language, json_path })}${args.code}` }
+                            : args,
+                    signal,
+                }),
+        };
+    }
+
+    return tool;
+}
+
+function codePrefix({ language, json_path }: { language?: unknown; json_path: string }) {
+    const path = JSON.stringify(json_path);
+    const lang = typeof language == "string" ? language.toLowerCase() : "python";
+
+    if (lang.startsWith("py")) {
+        return `import json as _cobble_json, os as _cobble_os; _cobble_os.environ.update(_cobble_json.load(open(${path})) if _cobble_os.path.exists(${path}) else {})\n`;
+    }
+
+    if (lang.startsWith("js") || lang.startsWith("javascript") || lang.startsWith("node")) {
+        return `try { Object.assign(process.env, JSON.parse(require("fs").readFileSync(${path}, "utf8"))); } catch {}\n`;
+    }
+
+    return "";
+}

src/harness/tools/computer/write-computer-env.ts

@@ -0,0 +1,62 @@
+import { AppData } from "@/src/data/app-data";
+import composeComputerEnvFiles from "./compose-computer-env-files";
+
+type Params = {
+    /**
+     * The sandbox's REST address (no `/mcp`).
+     */
+    base_url: string;
+    envs: { key: string; value: string }[];
+    signal?: AbortSignal;
+};
+
+/**
+ * Writes the computer's env files (`~/.cobble/env` and `env.json`, owner
+ * only), replacing the last run's, so removed envs disappear too. Returns
+ * their paths for `withComputerEnv`.
+ */
+export default async function writeComputerEnv({ base_url, envs, signal }: Params) {
+    const call = async (path: string, init?: RequestInit) => {
+        const res = await fetch(`${base_url}${path}`, {
+            ...init,
+            headers: { "content-type": "application/json" },
+            signal: AbortSignal.any([
+                ...(signal ? [signal] : []),
+                AbortSignal.timeout(AppData.McpConnectTimeoutMs),
+            ]),
+        });
+        const body: any = await res.json().catch(() => ({}));
+
+        if (!res.ok || body?.success === false) {
+            throw new Error(`The computer refused ${path}: ${body?.message || res.status}`);
+        }
+
+        return body;
+    };
+
+    const context = await call(`/v1/sandbox`);
+    const home = String(context.home_dir || "/root").replace(/\/+$/, "");
+    const dir = `${home}/.cobble`;
+    const shell_path = `${dir}/env`;
+    const json_path = `${dir}/env.json`;
+    const files = composeComputerEnvFiles({ envs });
+
+    await call(`/v1/shell/exec`, {
+        method: "POST",
+        body: JSON.stringify({
+            command: `mkdir -p ${dir} && chmod 700 ${dir} && : > ${shell_path} && : > ${json_path} && chmod 600 ${shell_path} ${json_path}`,
+        }),
+    });
+
+    for (const [file, content] of [
+        [shell_path, files.shell],
+        [json_path, files.json],
+    ] as const) {
+        await call(`/v1/file/write`, {
+            method: "POST",
+            body: JSON.stringify({ file, content }),
+        });
+    }
+
+    return { shell_path, json_path };
+}

src/harness/tools/run-tool-call.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, test } from "bun:test";
+import { z } from "zod";
+import { AppData } from "@/src/data/app-data";
+import HarnessError from "../harness-error";
+import type { HarnessTool } from "../types";
+import runToolCall from "./run-tool-call";
+import toolDefinition from "./tool-definition";
+
+const echo: HarnessTool<{ text: string }> = {
+    name: "echo",
+    description: "Says it back.",
+    schema: z.object({ text: z.string().min(1) }),
+    run: async ({ args }) => args.text,
+};
+
+const failing: HarnessTool = {
+    name: "fail",
+    description: "Always fails.",
+    schema: z.object({}),
+    run: async () => {
+        throw new Error("disk full");
+    },
+};
+
+const signal = new AbortController().signal;
+const call = (name: string, args: string) =>
+    runToolCall({
+        call: { id: "1", name, arguments: args },
+        tools: [echo, failing],
+        signal,
+    });
+
+describe("runToolCall", () => {
+    test("runs the tool with validated arguments", async () => {
+        expect(await call("echo", `{"text":"hi"}`)).toBe("hi");
+    });
+
+    test("reports problems to the model instead of throwing", async () => {
+        expect(await call("nope", "{}")).toContain(`no tool called "nope"`);
+        expect(await call("echo", "{bad")).toContain("weren't valid JSON");
+        expect(await call("echo", `{"text":""}`)).toContain(
+            "Wrong arguments for echo",
+        );
+        expect(await call("fail", "")).toBe("Error: disk full");
+    });
+
+    test("cuts long results", async () => {
+        const long = "x".repeat(AppData.HarnessToolResultMaxChars + 5);
+        const result = await call("echo", JSON.stringify({ text: long }));
+
+        expect(result).toEndWith("[Cut: 5 more characters.]");
+    });
+
+    test("throws when cancelled", async () => {
+        const controller = new AbortController();
+        const slow: HarnessTool = {
+            ...failing,
+            run: async () => {
+                controller.abort();
+                throw new Error("stopped");
+            },
+        };
+
+        const error = await runToolCall({
+            call: { id: "1", name: "fail", arguments: "{}" },
+            tools: [slow],
+            signal: controller.signal,
+        }).catch((e) => e);
+
+        expect(error).toBeInstanceOf(HarnessError);
+        expect(error.code).toBe("aborted");
+    });
+});
+
+describe("toolDefinition", () => {
+    test("describes the arguments as JSON schema", () => {
+        expect(toolDefinition({ tool: echo })).toEqual({
+            name: "echo",
+            description: "Says it back.",
+            parameters: {
+                type: "object",
+                properties: { text: { type: "string", minLength: 1 } },
+                required: ["text"],
+            },
+        });
+    });
+});

src/harness/tools/run-tool-call.ts

@@ -0,0 +1,61 @@
+import { z } from "zod";
+import { AppData } from "@/src/data/app-data";
+import HarnessError from "../harness-error";
+import type { HarnessTool, HarnessToolCall } from "../types";
+
+type Params = {
+    call: HarnessToolCall;
+    tools: HarnessTool[];
+    signal: AbortSignal;
+};
+
+/**
+ * Runs one tool call and returns the text the model gets back. Unknown
+ * tools, bad arguments and tool failures come back as `Error: ...` so the
+ * model can correct itself; only cancellation is thrown.
+ */
+export default async function runToolCall({ call, tools, signal }: Params) {
+    const tool = tools.find((t) => t.name == call.name);
+
+    if (!tool) {
+        return `Error: There is no tool called "${call.name}". Available: ${tools.map((t) => t.name).join(", ") || "none"}.`;
+    }
+
+    let raw: unknown;
+
+    try {
+        raw = call.arguments.trim() ? JSON.parse(call.arguments) : {};
+    } catch {
+        return `Error: The arguments for ${call.name} weren't valid JSON. Send a JSON object.`;
+    }
+
+    if (!raw || typeof raw != "object" || Array.isArray(raw)) {
+        return `Error: The arguments for ${call.name} must be a JSON object.`;
+    }
+
+    const parsed = tool.schema?.safeParse(raw);
+
+    if (parsed && !parsed.success) {
+        return `Error: Wrong arguments for ${call.name}.\n${z.prettifyError(parsed.error)}`;
+    }
+
+    try {
+        return truncate({
+            text: await tool.run({ args: parsed ? parsed.data : raw, signal }),
+        });
+    } catch (error: unknown) {
+        if (signal.aborted) {
+            throw new HarnessError({ message: `Cancelled.`, code: "aborted" });
+        }
+
+        return `Error: ${error instanceof Error ? error.message : String(error)}`;
+    }
+}
+
+function truncate({ text }: { text: string }) {
+    const max = AppData.HarnessToolResultMaxChars;
+
+    if (text.length <= max) return text;
+
+    return `${text.slice(0, max)}\n[Cut: ${text.length - max} more characters.]`;
+}

src/harness/tools/tool-definition.ts

@@ -0,0 +1,24 @@
+import { z } from "zod";
+import type { HarnessTool, HarnessToolDefinition } from "../types";
+
+type Params = {
+    tool: HarnessTool;
+};
+
+/**
+ * What the model sees of a tool: its name, description and arguments as
+ * JSON schema (draft 7, the one providers accept most widely).
+ */
+export default function toolDefinition({ tool }: Params): HarnessToolDefinition {
+    const { $schema, ...parameters } = (
+        tool.schema
+            ? z.toJSONSchema(tool.schema, { target: "draft-7", io: "input" })
+            : tool.parameters || { type: "object", properties: {} }
+    ) as Record<string, unknown>;
+
+    return {
+        name: tool.name,
+        description: tool.description,
+        parameters,
+    };
+}

src/harness/types.ts

@@ -0,0 +1,219 @@
+import type { z } from "zod";
+import type { HarnessKind } from "@/src/dict/harness-dict";
+import type { HarnessProviderKind } from "@/src/dict/harness-provider-dict";
+
+/**
+ * Part of a user message. Images are an https or `data:` URL; audio is
+ * base64 with its format (`mp3`, `wav`, `webm`, `m4a`, ...).
+ */
+export type HarnessContentPart =
+    | { type: "text"; text: string }
+    | { type: "image"; url: string; name?: string }
+    | { type: "audio"; data: string; format: string; name?: string };
+
+export type HarnessToolCall = {
+    id: string;
+    name: string;
+    /**
+     * JSON text, exactly as the model wrote it.
+     */
+    arguments: string;
+};
+
+export type HarnessMessage =
+    | { role: "system"; content: string }
+    | { role: "user"; content: string | HarnessContentPart[] }
+    | {
+          role: "assistant";
+          content: string | null;
+          tool_calls?: HarnessToolCall[];
+      }
+    | { role: "tool"; tool_call_id: string; content: string };
+
+/**
+ * What the model sees of a tool.
+ */
+export type HarnessToolDefinition = {
+    name: string;
+    description: string;
+    /**
+     * JSON schema of the arguments object.
+     */
+    parameters: Record<string, unknown>;
+};
+
+/**
+ * A tool the text harness can call. Tools are built per run with whatever
+ * they act on (bot, user, computer) already bound, so `run` only gets the
+ * arguments. Return text for the model; throw to report a failure to it.
+ *
+ * Own tools describe their arguments with a zod `schema`, checked before
+ * `run`. Tools described elsewhere (an MCP server) pass its JSON schema as
+ * `parameters` instead and check their own arguments.
+ */
+export type HarnessTool<Args = any> = {
+    name: string;
+    description: string;
+    schema?: z.ZodType<Args>;
+    parameters?: Record<string, unknown>;
+    run: (params: { args: Args; signal: AbortSignal }) => Promise<string>;
+};
+
+/**
+ * A provider as the adapters need it: where to call and with which key.
+ */
+export type HarnessProviderConfig = {
+    id?: number;
+    kind: HarnessProviderKind;
+    name?: string;
+    base_url: string;
+    api_key?: string;
+};
+
+/**
+ * What the harness knows about a model, from the catalog.
+ */
+export type HarnessModelInfo = {
+    slug: string;
+    name?: string;
+    input_modalities: string[];
+    output_modalities: string[];
+    supports_tools: boolean;
+    context_length?: number;
+    max_output_tokens?: number;
+    voices?: string[];
+};
+
+/**
+ * A catalog entry as a provider lists it.
+ */
+export type HarnessCatalogModel = HarnessModelInfo & {
+    free: boolean;
+    /**
+     * Unix ms.
+     */
+    released_at?: number;
+    /**
+     * Unix ms. The provider retires the model then.
+     */
+    expires_at?: number;
+    /**
+     * Artificial Analysis intelligence index, when the provider shares it.
+     */
+    intelligence_score?: number;
+    /**
+     * Artificial Analysis agentic index (tool use, multi-step work).
+     */
+    agentic_score?: number;
+};
+
+/**
+ * One provider + model a harness can call, in the order they are tried.
+ */
+export type HarnessCandidate = {
+    /**
+     * `<provider id>:<slug>`. Identifies the candidate for cooldowns.
+     */
+    key: string;
+    provider: HarnessProviderConfig;
+    model: HarnessModelInfo;
+    /**
+     * `pinned`: the super admin's pick. `auto`: an automatic free pick.
+     */
+    source: "pinned" | "auto";
+};
+
+export type HarnessUsage = {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    /**
+     * In the provider's currency (USD for OpenRouter), when reported.
+     */
+    cost?: number;
+};
+
+export type HarnessChatParams = {
+    candidate: HarnessCandidate;
+    messages: HarnessMessage[];
+    tools?: HarnessToolDefinition[];
+    max_tokens?: number;
+    temperature?: number;
+    signal: AbortSignal;
+};
+
+export type HarnessChatResult = {
+    text: string | null;
+    tool_calls: HarnessToolCall[];
+    finish_reason?: string;
+    usage?: HarnessUsage;
+};
+
+export type HarnessAudioInput = {
+    data: Uint8Array;
+    /**
+     * `mp3`, `wav`, `webm`, `m4a`, `ogg`, `flac`.
+     */
+    format: string;
+};
+
+export type HarnessTranscribeParams = {
+    candidate: HarnessCandidate;
+    audio: HarnessAudioInput;
+    language?: string;
+    signal: AbortSignal;
+};
+
+export type HarnessSpeakParams = {
+    candidate: HarnessCandidate;
+    text: string;
+    voice?: string;
+    /**
+     * `mp3` unless set.
+     */
+    format?: string;
+    signal: AbortSignal;
+};
+
+export type HarnessAudioOutput = {
+    data: Uint8Array;
+    media_type: string;
+};
+
+export type HarnessImageParams = {
+    candidate: HarnessCandidate;
+    prompt: string;
+    n?: number;
+    /**
+     * e.g. `1024x1024`, when the model takes one.
+     */
+    size?: string;
+    /**
+     * Reference images: https or `data:` URLs.
+     */
+    references?: string[];
+    signal: AbortSignal;
+};
+
+export type HarnessImageOutput = {
+    data: Uint8Array;
+    media_type: string;
+};
+
+/**
+ * How the harness talks to one kind of provider. Every call takes the
+ * candidate it runs on and an abort signal.
+ */
+export type HarnessAdapter = {
+    chat: (params: HarnessChatParams) => Promise<HarnessChatResult>;
+    transcribe: (params: HarnessTranscribeParams) => Promise<{ text: string }>;
+    speak: (params: HarnessSpeakParams) => Promise<HarnessAudioOutput>;
+    generateImage: (
+        params: HarnessImageParams,
+    ) => Promise<{ images: HarnessImageOutput[] }>;
+    listModels: (params: {
+        provider: HarnessProviderConfig;
+        signal?: AbortSignal;
+    }) => Promise<HarnessCatalogModel[]>;
+};
+
+export type { HarnessKind };