Add the harness, envs, per-bot models and chat attachments

- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-19 03:38:38 +01:00
parent d49f524fe8
commit 6a2665dae9
223 changed files with 12342 additions and 162 deletions
@@ -9,3 +9,25 @@ export const BotJobCancelledError = "Cancelled";
 * silent past the lock timeout on its last attempt.
 */
 export const BotJobLockTimedOutError = "Lock timed out";
+
+/**
+ * What a queued job is. A `task` is long-running work with a row in the
+ * working-on panel; a `reply` answers the user's latest messages in chat.
+ */
+export const BotJobKinds = [
+    { title: "Task", value: "task" },
+    { title: "Reply", value: "reply" },
+] as const;
+
+export type BotJobKind = (typeof BotJobKinds)[number]["value"];
+
+/**
+ * A reply job's title, and the bot's status line while it runs. The chat
+ * shows a thinking row while a bot has it.
+ */
+export const BotReplyJobTitle = "Thinking";
+
+/**
+ * The bot's status line while its job waits for a rate-limited model.
+ */
+export const BotWaitingStatusLine = "Waiting for a free model";
@@ -70,3 +70,20 @@ export const ContainerPurposes = [
 ] as const;

 export type ContainerPurpose = (typeof ContainerPurposes)[number]["value"];
+
+/**
+ * Path of the AIO sandbox's MCP endpoint, on the computer's address.
+ */
+export const ComputerMcpPath = "/mcp";
+
+/**
+ * Sandbox MCP tools bots don't get: noise (cdp urls), skills they have no
+ * use for, and raw display control that needs pixel coordinates (the page
+ * tools cover the same ground by element).
+ */
+export const ComputerMcpExcludedTools = [
+    "browser_get_info",
+    "sandbox_load_skill",
+    "browser_gui_screenshot",
+    "browser_gui_execute_action",
+] as const;
@@ -0,0 +1,79 @@
+import { HarnessProviderKinds } from "./harness-provider-dict";
+
+/**
+ * Where an env applies. A computer's env beats a workspace's, which beats
+ * a general one.
+ */
+export const EnvParadigms = [
+    {
+        title: "General",
+        value: "general",
+    },
+    {
+        title: "Workspace",
+        value: "workspace",
+    },
+    {
+        title: "Computer",
+        value: "computer",
+    },
+] as const;
+
+export type EnvParadigm = (typeof EnvParadigms)[number]["value"];
+
+export type KnownEnv = {
+    key: string;
+    description: string;
+    required: boolean;
+    /**
+     * Hidden once saved; only replaced, never shown.
+     */
+    secret: boolean;
+};
+
+/**
+ * Envs the app itself reads (`grabEnv`), set in Settings, under
+ * Environment. They are general only and never go to computers. Users add
+ * their own keys beside these for bots and computers to use.
+ *
+ * Not here, and never in the `envs` table: `ENCRYPTION_KEY` and
+ * `ENCRYPTION_SALT` (they encrypt the table), and `PORT`, `DATA_DIR` and
+ * `MAIN_SERVER` (read before the database opens). Those stay in the
+ * process environment.
+ */
+export const EnvDict: readonly KnownEnv[] = [
+    {
+        key: "MAIN_COMPUTER_URL",
+        description: `The main computer's address, for a server running outside docker (e.g. http://localhost:8170). Empty uses the docker network.`,
+        required: false,
+        secret: false,
+    },
+    ...HarnessProviderKinds.flatMap((provider) =>
+        provider.key_env
+            ? [
+                  {
+                      key: provider.key_env,
+                      description: `${provider.title} key, used when the provider has no key of its own.`,
+                      required: false,
+                      secret: true,
+                  },
+              ]
+            : [],
+    ),
+];
+
+/**
+ * Keys that must stay in the process environment.
+ */
+export const ProcessOnlyEnvKeys = [
+    "ENCRYPTION_KEY",
+    "ENCRYPTION_SALT",
+    "PORT",
+    "DATA_DIR",
+    "MAIN_SERVER",
+] as const;
+
+/**
+ * What an env key may look like: a shell variable name.
+ */
+export const EnvKeyRegex = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/;
@@ -0,0 +1,69 @@
+/**
+ * The harnesses: what a model is used for. Each harness has one model
+ * setting (`harness_settings`), automatic by default. Text is the brain and
+ * can't be turned off; the others are helpers it calls on.
+ */
+export const Harnesses = [
+    {
+        title: "Text",
+        value: "text",
+        description: `Chats, plans and works on computers. Every bot thinks with it.`,
+        required: true,
+    },
+    {
+        title: "Media reading",
+        value: "vision",
+        description: `Reads images, scans and screenshots for the text model when it can't see them itself.`,
+        required: false,
+    },
+    {
+        title: "Transcription",
+        value: "transcription",
+        description: `Turns voice messages and recordings into text.`,
+        required: false,
+    },
+    {
+        title: "Speech",
+        value: "speech",
+        description: `Reads bot replies aloud.`,
+        required: false,
+    },
+    {
+        title: "Image generation",
+        value: "image",
+        description: `Makes images when a bot is asked for one.`,
+        required: false,
+    },
+] as const;
+
+export type HarnessKind = (typeof Harnesses)[number]["value"];
+
+/**
+ * Vendors preferred for automatic picks, best first. Free models from any
+ * other vendor come after these, and only if they look competent.
+ */
+export const AutoModelVendorPreference = [
+    "deepseek",
+    "qwen",
+    "nvidia",
+    "liquid",
+] as const;
+
+/**
+ * Routers that pick a model themselves (e.g. `openrouter/free`). Used as
+ * the last automatic pick, after every named model.
+ */
+export const AutoModelRouterSlugs = ["openrouter/free"];
+
+/**
+ * Models with one narrow job (moderation, embeddings, ranking). Never
+ * picked automatically.
+ */
+export const SpecialistModelPattern =
+    /guard|safety|moderation|embed|rerank|classif/i;
+
+/**
+ * Names of small variants, ranked after their bigger siblings.
+ */
+export const SmallModelPattern =
+    /(?:^|[-_.:/])(nano|mini|small|lite|lightning|tiny|micro|xs)(?=$|[-_.:])/i;
@@ -0,0 +1,123 @@
+/**
+ * Model providers a super admin can plug in. Every one is called through
+ * its OpenAI-compatible API for now (`src/harness/providers/`); a provider
+ * that needs its own API gets its own adapter there.
+ *
+ * - `key_env`: environment variable used when no key is saved
+ * - `catalog`: how its models are listed. `openrouter` gives modalities,
+ *   tools support and prices; `openai-compatible` gives only ids, so
+ *   capabilities are guessed from the names
+ * - `local`: runs on the user's own hardware, so its models count as free
+ * - `image_path`: image generation endpoint
+ */
+export const HarnessProviderKinds = [
+    {
+        title: "OpenRouter",
+        value: "openrouter",
+        base_url: "https://openrouter.ai/api/v1",
+        needs_key: true,
+        key_env: "OPENROUTER_API_KEY",
+        catalog: "openrouter",
+        local: false,
+        image_path: "/images",
+    },
+    {
+        title: "OpenAI",
+        value: "openai",
+        base_url: "https://api.openai.com/v1",
+        needs_key: true,
+        key_env: "OPENAI_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Anthropic",
+        value: "anthropic",
+        base_url: "https://api.anthropic.com/v1",
+        needs_key: true,
+        key_env: "ANTHROPIC_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Google Gemini",
+        value: "google",
+        base_url: "https://generativelanguage.googleapis.com/v1beta/openai",
+        needs_key: true,
+        key_env: "GEMINI_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "DeepSeek",
+        value: "deepseek",
+        base_url: "https://api.deepseek.com",
+        needs_key: true,
+        key_env: "DEEPSEEK_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Groq",
+        value: "groq",
+        base_url: "https://api.groq.com/openai/v1",
+        needs_key: true,
+        key_env: "GROQ_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Mistral",
+        value: "mistral",
+        base_url: "https://api.mistral.ai/v1",
+        needs_key: true,
+        key_env: "MISTRAL_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "xAI",
+        value: "xai",
+        base_url: "https://api.x.ai/v1",
+        needs_key: true,
+        key_env: "XAI_API_KEY",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Ollama",
+        value: "ollama",
+        base_url: "http://localhost:11434/v1",
+        needs_key: false,
+        key_env: "",
+        catalog: "openai-compatible",
+        local: true,
+        image_path: "/images/generations",
+    },
+    {
+        title: "Other (OpenAI-compatible)",
+        value: "custom",
+        base_url: "",
+        needs_key: false,
+        key_env: "",
+        catalog: "openai-compatible",
+        local: false,
+        image_path: "/images/generations",
+    },
+] as const;
+
+export type HarnessProviderKind = (typeof HarnessProviderKinds)[number]["value"];
+
+export type HarnessProviderPreset = (typeof HarnessProviderKinds)[number];
+
+/**
+ * Created on first run so the automatic free picks work out of the box.
+ */
+export const DefaultHarnessProviderKind: HarnessProviderKind = "openrouter";
@@ -11,6 +11,10 @@ export const MediaTypes = [
        title: "Document",
        value: "document",
    },
+    {
+        title: "Audio",
+        value: "audio",
+    },
 ] as const;

 export const MediaParadigms = [
@@ -26,4 +30,8 @@ export const MediaParadigms = [
        title: "Event",
        value: "event",
    },
+    {
+        title: "Bot Attachment",
+        value: "bot-attachment",
+    },
 ] as const;
@@ -1,7 +1,13 @@
 import type { TableType } from "@/src/types";

 /**
- * Tables holding secrets (password hashes, pending login codes). They are
- * never reachable through the admin CRUD API, not even by super admins.
+ * Tables holding secrets (password hashes, pending login codes, model
+ * provider keys, envs). They are never reachable through the admin CRUD API, not
+ * even by super admins.
 */
-export const PrivateTables: TableType[] = ["user_passwords", "sso_login_codes"];
+export const PrivateTables: TableType[] = [
+    "user_passwords",
+    "sso_login_codes",
+    "harness_providers",
+    "envs",
+];
@@ -1,7 +1,9 @@
 import {
+    CpuAi,
    KeyRound,
    SlidersHorizontal,
    UserRound,
+    Variable,
    type LucideIcon,
 } from "lucide-react";

@@ -9,6 +11,10 @@ export type SettingsPage = {
    title: string;
    url: string;
    Icon: LucideIcon;
+    /**
+     * Shown and served to super admins only.
+     */
+    super_admin_only?: boolean;
 };

 /**
@@ -27,6 +33,18 @@ export const SettingsPages: SettingsPage[] = [
        url: "/admin/settings/change-password",
        Icon: KeyRound,
    },
+    {
+        title: "Models",
+        url: "/admin/settings/models",
+        Icon: CpuAi,
+        super_admin_only: true,
+    },
+    {
+        title: "Environment",
+        url: "/admin/settings/environment",
+        Icon: Variable,
+        super_admin_only: true,
+    },
 ];

 export const SettingsDefaultPath = "/admin/settings/general";