Add the harness, envs, per-bot models and chat attachments

- Harness: providers, model catalog synced hourly, automatic free picks,
  cooldowns, tool loop with context trimming, MCP computer tools
- Bots queue runs replies and tasks through the harness; jobs wait
  (attempt refunded) while every model is resting
- Envs: Settings > Environment, scoped general/workspace/computer,
  written to computers and redacted from tool output
- Models: Settings > Models, per-harness provider select defaulting to
  OpenRouter, and a models modal on each bot's chat (the coordinator's
  edits the defaults)
- Chat: thinking row while a reply runs, file attachments (upload,
  paste, drop) that replies read as images, audio and document text
- Fix media uploads failing after a user's first file, and private
  files being served from the public media path

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
2026-10-19 03:38:38 +01:00
co-authored by Claude Opus 5.5
parent d49f524fe8
commit 6a2665dae9
223 changed files with 12342 additions and 162 deletions
@@ -0,0 +1,41 @@
import { describe, expect, test } from "bun:test";
import guessModelModalities from "./guess-model-modalities";
const outputs = (slug: string) =>
guessModelModalities({ slug }).output_modalities;
describe("guessModelModalities", () => {
test("spots speech, transcription, image and embedding models", () => {
expect(outputs("whisper-large-v3")).toEqual(["transcription"]);
expect(outputs("gpt-4o-mini-transcribe")).toEqual(["transcription"]);
expect(outputs("tts-1-hd")).toEqual(["speech"]);
expect(outputs("gpt-image-1")).toEqual(["image"]);
expect(outputs("dall-e-3")).toEqual(["image"]);
expect(outputs("text-embedding-3-small")).toEqual(["embeddings"]);
});
test("takes everything else for a tool-calling chat model", () => {
expect(guessModelModalities({ slug: "deepseek-chat" })).toEqual({
input_modalities: ["text"],
output_modalities: ["text"],
supports_tools: true,
});
expect(outputs("mistral-large-latest")).toEqual(["text"]);
});
test("knows which chat families see images", () => {
expect(
guessModelModalities({ slug: "claude-sonnet-5-5" }).input_modalities,
).toEqual(["text", "image"]);
expect(
guessModelModalities({ slug: "models/gemini-3.5-flash" })
.input_modalities,
).toEqual(["text", "image"]);
expect(
guessModelModalities({ slug: "qwen2.5-vl:7b" }).input_modalities,
).toEqual(["text", "image"]);
expect(
guessModelModalities({ slug: "deepseek-r1:14b" }).input_modalities,
).toEqual(["text"]);
});
});
@@ -0,0 +1,65 @@
import type { HarnessModelInfo } from "../types";
type Params = {
slug: string;
};
type Guess = Pick<
HarnessModelInfo,
"input_modalities" | "output_modalities" | "supports_tools"
>;
const Transcription = /whisper|transcri|(?:^|[-_/])(stt|asr)(?=$|[-_:.])|scribe/i;
const Speech = /(?:^|[-_/])tts(?=$|[-_:.])|speech|voice/i;
const ImageOut = /dall-e|gpt-image|imagen|flux|stable-diffusion|sdxl|image-gen/i;
const Embedding = /embed/i;
/**
* Chat families that read images.
*/
const ImageIn =
/vision|(?:^|[-_/])vl(?=$|[-_:.])|gpt-4o|gpt-4\.1|gpt-[5-9]|claude|gemini|llava|pixtral|gemma-?[34]|grok-[2-9]|llama-?4|mistral-(small|medium)/i;
/**
* What a model takes and makes, guessed from its id, for providers whose
* model list only has ids. Anything that isn't plainly a speech, image or
* embedding model is taken for a chat model that can call tools.
*/
export default function guessModelModalities({ slug }: Params): Guess {
if (Transcription.test(slug)) {
return {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
};
}
if (Speech.test(slug)) {
return {
input_modalities: ["text"],
output_modalities: ["speech"],
supports_tools: false,
};
}
if (ImageOut.test(slug)) {
return {
input_modalities: ["text", "image"],
output_modalities: ["image"],
supports_tools: false,
};
}
if (Embedding.test(slug)) {
return {
input_modalities: ["text"],
output_modalities: ["embeddings"],
supports_tools: false,
};
}
return {
input_modalities: ImageIn.test(slug) ? ["text", "image"] : ["text"],
output_modalities: ["text"],
supports_tools: true,
};
}
@@ -0,0 +1,75 @@
import { describe, expect, test } from "bun:test";
import modelServesHarness from "./model-serves-harness";
const chat = {
input_modalities: ["text"],
output_modalities: ["text"],
supports_tools: true,
};
describe("modelServesHarness", () => {
test("text needs tools", () => {
expect(modelServesHarness({ model: chat, harness: "text" })).toBe(true);
expect(
modelServesHarness({
model: { ...chat, supports_tools: false },
harness: "text",
}),
).toBe(false);
});
test("vision needs image input", () => {
expect(modelServesHarness({ model: chat, harness: "vision" })).toBe(
false,
);
expect(
modelServesHarness({
model: { ...chat, input_modalities: ["text", "image"] },
harness: "vision",
}),
).toBe(true);
});
test("transcription takes speech-to-text or audio-hearing chat models", () => {
expect(
modelServesHarness({
model: {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
},
harness: "transcription",
}),
).toBe(true);
expect(
modelServesHarness({
model: { ...chat, input_modalities: ["text", "audio"] },
harness: "transcription",
}),
).toBe(true);
expect(
modelServesHarness({ model: chat, harness: "transcription" }),
).toBe(false);
});
test("speech and image need that output", () => {
expect(
modelServesHarness({
model: { ...chat, output_modalities: ["speech"] },
harness: "speech",
}),
).toBe(true);
expect(
modelServesHarness({
model: { ...chat, output_modalities: ["text", "audio"] },
harness: "speech",
}),
).toBe(false);
expect(
modelServesHarness({
model: { ...chat, output_modalities: ["image", "text"] },
harness: "image",
}),
).toBe(true);
});
});
@@ -0,0 +1,45 @@
import type { HarnessKind } from "@/src/dict/harness-dict";
import type { HarnessModelInfo } from "../types";
type Params = {
model: Pick<
HarnessModelInfo,
"input_modalities" | "output_modalities" | "supports_tools"
>;
harness: HarnessKind;
};
/**
* Whether a model can do a harness's job:
* - text: reads and writes text, and calls tools (bots work through tools)
* - vision: reads images, writes text
* - transcription: a speech-to-text model, or a chat model that hears audio
* - speech: a text-to-speech model
* - image: makes images
*/
export default function modelServesHarness({ model, harness }: Params) {
const input = new Set(model.input_modalities);
const output = new Set(model.output_modalities);
switch (harness) {
case "text":
return (
input.has("text") && output.has("text") && model.supports_tools
);
case "vision":
return input.has("image") && output.has("text");
case "transcription":
return (
output.has("transcription") ||
(input.has("audio") && output.has("text"))
);
case "speech":
return output.has("speech");
case "image":
return output.has("image");
}
}
@@ -0,0 +1,26 @@
import { describe, expect, test } from "bun:test";
import parseModelSize from "./parse-model-size";
describe("parseModelSize", () => {
test("reads total parameters, skipping active ones", () => {
expect(parseModelSize({ slug: "qwen/qwen3-30b-a3b" })).toBe(30);
expect(
parseModelSize({ slug: "nvidia/nemotron-3-ultra-550b-a55b:free" }),
).toBe(550);
expect(parseModelSize({ slug: "liquid/lfm-2.5-2.6b:free" })).toBe(2.6);
expect(parseModelSize({ slug: "deepseek-r1:14b" })).toBe(14);
});
test("reads trillions", () => {
expect(parseModelSize({ slug: "qwen/qwen3.8-2.4t-a95b" })).toBe(2400);
});
test("is undefined when the slug has no size", () => {
expect(parseModelSize({ slug: "deepseek/deepseek-v4.1-flash" })).toBe(
undefined,
);
expect(parseModelSize({ slug: "google/gemma-3n-e4b-it" })).toBe(
undefined,
);
});
});
+19
View File
@@ -0,0 +1,19 @@
type Params = {
slug: string;
};
/**
* Total parameters in billions, read from a model slug
* (`qwen3-30b-a3b` is 30, `qwen3.8-2.4t-a95b` is 2400). Active-parameter
* counts (`a3b`) are skipped. Undefined when the slug doesn't say.
*/
export default function parseModelSize({ slug }: Params): number | undefined {
const name = slug.split("/").pop() || slug;
const match = name.match(/(?<![a-z0-9.])(\d+(?:\.\d+)?)([bt])(?![a-z])/i);
if (!match?.[1] || !match[2]) return undefined;
const size = Number(match[1]);
return match[2].toLowerCase() == "t" ? size * 1000 : size;
}
@@ -0,0 +1,165 @@
import { describe, expect, test } from "bun:test";
import type { HarnessCatalogModel } from "../types";
import rankAutoModels from "./rank-auto-models";
const day = 24 * 60 * 60 * 1000;
function model(
slug: string,
extra: Partial<HarnessCatalogModel> = {},
): HarnessCatalogModel {
return {
slug,
free: true,
input_modalities: ["text"],
output_modalities: ["text"],
supports_tools: true,
context_length: 262144,
released_at: 0,
...extra,
};
}
/**
* OpenRouter's free list on 2026-10-18 (no DeepSeek or Qwen that day).
*/
const live = [
model("xingchen-agi/xing4.0-29b-a4b:free", { released_at: 20 * day }),
model("nvidia/nemotron-3.5-super-vl:free", {
input_modalities: ["text", "image", "video"],
intelligence_score: 26.5,
}),
model("liquid/lfm-2.5-2.6b:free", {
context_length: 65536,
intelligence_score: 8.4,
}),
model("nvidia/nemotron-3.5-lightning:free", { intelligence_score: 12.9 }),
model("thinkingmachines/inkling-small:free", {
input_modalities: ["text", "image", "audio"],
intelligence_score: 25.7,
agentic_score: 23.5,
}),
model("cohere/north-mini-code:free", { intelligence_score: 9.9 }),
model("nvidia/nemotron-3.5-content-safety:free", {
supports_tools: false,
}),
model("nvidia/nemotron-3-ultra-550b-a55b:free", {
intelligence_score: 22.9,
}),
model("google/gemma-4-31b-it:free", {
input_modalities: ["image", "text", "video"],
intelligence_score: 14.7,
}),
model("openrouter/free", { input_modalities: ["text", "image"] }),
model("anthropic/claude-paid", { free: false }),
];
const slugs = (models: HarnessCatalogModel[]) => models.map((m) => m.slug);
describe("rankAutoModels", () => {
test("follows the vendor preference, then competent others, then routers", () => {
expect(slugs(rankAutoModels({ models: live, harness: "text" }))).toEqual(
[
"nvidia/nemotron-3.5-super-vl:free",
"nvidia/nemotron-3-ultra-550b-a55b:free",
"nvidia/nemotron-3.5-lightning:free",
"liquid/lfm-2.5-2.6b:free",
"thinkingmachines/inkling-small:free",
"google/gemma-4-31b-it:free",
"xingchen-agi/xing4.0-29b-a4b:free",
"openrouter/free",
],
);
});
test("puts DeepSeek, then Qwen, ahead of everything once they're free", () => {
const ranked = rankAutoModels({
models: [
...live,
model("qwen/qwen3.8-flash:free"),
model("deepseek/deepseek-v4.1-flash:free"),
],
harness: "text",
});
expect(slugs(ranked).slice(0, 2)).toEqual([
"deepseek/deepseek-v4.1-flash:free",
"qwen/qwen3.8-flash:free",
]);
});
test("recognises vendors in local model names", () => {
const ranked = rankAutoModels({
models: [model("llama3.3:70b"), model("qwen3:32b")],
harness: "text",
});
expect(slugs(ranked)).toEqual(["qwen3:32b", "llama3.3:70b"]);
});
test("filters by what the harness needs", () => {
expect(
slugs(rankAutoModels({ models: live, harness: "vision" })),
).toEqual([
"nvidia/nemotron-3.5-super-vl:free",
"thinkingmachines/inkling-small:free",
"google/gemma-4-31b-it:free",
"openrouter/free",
]);
expect(
slugs(rankAutoModels({ models: live, harness: "transcription" })),
).toEqual(["thinkingmachines/inkling-small:free"]);
expect(rankAutoModels({ models: live, harness: "speech" })).toEqual(
[],
);
});
test("puts speech-to-text models ahead of chat models that hear audio", () => {
const ranked = rankAutoModels({
models: [
model("deepseek/omni", { input_modalities: ["text", "audio"] }),
model("acme/scribe", {
input_modalities: ["audio"],
output_modalities: ["transcription"],
supports_tools: false,
}),
],
harness: "transcription",
});
expect(slugs(ranked)).toEqual(["acme/scribe", "deepseek/omni"]);
});
test("drops retired and unscored small models from other vendors", () => {
const ranked = rankAutoModels({
models: [
model("acme/tiny-7b"),
model("acme/big-mini"),
model("acme/gone", { expires_at: 10 }),
model("acme/short", { context_length: 8000 }),
model("acme/solid-70b"),
],
harness: "text",
now: 100,
});
expect(slugs(ranked)).toEqual(["acme/solid-70b"]);
});
test("within a vendor, unscored full-size models beat small ones, newest first", () => {
const ranked = rankAutoModels({
models: [
model("deepseek/deepseek-lite", { released_at: 3 }),
model("deepseek/deepseek-old", { released_at: 1 }),
model("deepseek/deepseek-new", { released_at: 2 }),
],
harness: "text",
});
expect(slugs(ranked)).toEqual([
"deepseek/deepseek-new",
"deepseek/deepseek-old",
"deepseek/deepseek-lite",
]);
});
});
+163
View File
@@ -0,0 +1,163 @@
import { AppData } from "@/src/data/app-data";
import {
AutoModelRouterSlugs,
AutoModelVendorPreference,
SmallModelPattern,
SpecialistModelPattern,
type HarnessKind,
} from "@/src/dict/harness-dict";
import type { HarnessCatalogModel } from "../types";
import modelServesHarness from "./model-serves-harness";
import parseModelSize from "./parse-model-size";
type Params<T extends HarnessCatalogModel> = {
models: T[];
harness: HarnessKind;
/**
* Unix ms, for retired models. Defaults to now.
*/
now?: number;
};
/**
* The free models a harness picks automatically, best first:
* 1. `AutoModelVendorPreference` in order (DeepSeek, Qwen, NVIDIA, Liquid)
* 2. any other vendor's model that looks competent (text and vision only)
* 3. routers such as `openrouter/free`, last
*
* Within a group, benchmarked models come first by intelligence index,
* then the rest: full-size before small variants, newest first.
*/
export default function rankAutoModels<T extends HarnessCatalogModel>({
models,
harness,
now = Date.now(),
}: Params<T>): T[] {
const ranked = models
.filter(
(model) =>
model.free &&
!(model.expires_at && model.expires_at <= now) &&
!SpecialistModelPattern.test(model.slug) &&
modelServesHarness({ model, harness }),
)
.map((model) => ({ model, tier: grabTier({ model, harness }) }))
.filter(
(entry): entry is { model: T; tier: number } =>
entry.tier !== undefined,
);
return ranked
.sort(
(a, b) =>
(harness == "transcription"
? Number(!isSpeechToText(a.model)) -
Number(!isSpeechToText(b.model))
: 0) ||
a.tier - b.tier ||
compareInTier(a.model, b.model),
)
.map((entry) => entry.model);
}
/**
* Dedicated speech-to-text models transcribe before chat models that hear
* audio: some free chat endpoints list audio input but drop it.
*/
function isSpeechToText(model: HarnessCatalogModel) {
return model.output_modalities.includes("transcription");
}
const OtherTier = AutoModelVendorPreference.length;
const RouterTier = OtherTier + 1;
/**
* Undefined drops the model.
*/
function grabTier({
model,
harness,
}: {
model: HarnessCatalogModel;
harness: HarnessKind;
}) {
const slug = model.slug.toLowerCase();
if (AutoModelRouterSlugs.includes(slug)) {
return RouterTier;
}
const name = slug.split("/").pop() || slug;
const vendor_tier = AutoModelVendorPreference.findIndex(
(vendor) => slug.startsWith(`${vendor}/`) || name.startsWith(vendor),
);
if (vendor_tier >= 0) {
return vendor_tier;
}
if (harness != "text" && harness != "vision") {
return OtherTier;
}
return isCompetent({ model }) ? OtherTier : undefined;
}
function isCompetent({ model }: { model: HarnessCatalogModel }) {
if (
model.context_length &&
model.context_length < AppData.HarnessAutoMinContext
) {
return false;
}
if (model.intelligence_score !== undefined) {
return model.intelligence_score >= AppData.HarnessAutoMinScore;
}
const size = parseModelSize({ slug: model.slug });
return (
!SmallModelPattern.test(model.slug) &&
(size === undefined || size >= AppData.HarnessAutoMinSizeB)
);
}
function compareInTier(a: HarnessCatalogModel, b: HarnessCatalogModel) {
const a_scored = a.intelligence_score !== undefined;
const b_scored = b.intelligence_score !== undefined;
if (a_scored != b_scored) {
return a_scored ? -1 : 1;
}
if (a_scored && b_scored) {
const by_score =
(b.intelligence_score || 0) - (a.intelligence_score || 0) ||
(b.agentic_score || 0) - (a.agentic_score || 0);
if (by_score) return by_score;
}
const a_small = isSmall(a);
const b_small = isSmall(b);
if (a_small != b_small) {
return a_small ? 1 : -1;
}
return (
(b.released_at || 0) - (a.released_at || 0) ||
(b.context_length || 0) - (a.context_length || 0) ||
a.slug.localeCompare(b.slug)
);
}
function isSmall(model: HarnessCatalogModel) {
const size = parseModelSize({ slug: model.slug });
return (
SmallModelPattern.test(model.slug) ||
(size !== undefined && size < AppData.HarnessAutoMinSizeB)
);
}