Add the harness, envs, per-bot models and chat attachments

- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-19 03:38:38 +01:00
parent d49f524fe8
commit 6a2665dae9
223 changed files with 12342 additions and 162 deletions
@@ -0,0 +1,88 @@
+import { ComputerMcpExcludedTools } from "@/src/dict/computer-dict";
+import grabComputerMcpUrl from "@/src/functions/backend/db/computers/grab-computer-mcp-url";
+import grabComputerEnvs from "@/src/functions/backend/db/envs/grab-computer-envs";
+import redactSecrets from "@/src/utils/redact-secrets";
+import connectMcpServer from "../../mcp/connect-mcp-server";
+import grabMcpTools from "../../mcp/grab-mcp-tools";
+import readImage from "../../media/read-image";
+import type { HarnessTool } from "../../types";
+import withComputerEnv from "./with-computer-env";
+import writeComputerEnv from "./write-computer-env";
+
+type Params = {
+    computer_id: number | string;
+    /**
+     * Screenshots are read with this bot's media reading model, and only
+     * workspaces it shares with the computer add envs.
+     */
+    bot_id?: number | string;
+    signal?: AbortSignal;
+};
+
+export type OpenComputer = {
+    name: string;
+    tools: HarnessTool[];
+    /**
+     * The envs set on the computer, for the prompt: keys and descriptions,
+     * never values.
+     */
+    envs: { key: string; description: string }[];
+    /**
+     * Blanks the computer's secret values out of text, e.g. the report.
+     */
+    redact: (text: string) => string;
+    close: () => Promise<void>;
+};
+
+/**
+ * Connects to a computer (its AIO sandbox, over MCP) and returns its tools:
+ * shell, files, code and browser. Its envs (`grabComputerEnvs`) are written
+ * to the computer and loaded by the shell and code tools; secret values are
+ * redacted from everything the tools return. Screenshots come back as
+ * readings from the media reading harness. Call `close` when the run ends.
+ * Throws when the computer can't be reached, which the queue retries.
+ */
+export default async function openComputerTools({
+    computer_id,
+    bot_id,
+    signal,
+}: Params): Promise<OpenComputer> {
+    const target = await grabComputerMcpUrl({ computer_id });
+
+    if (!target) {
+        throw new Error(`Computer #${computer_id} has no container to connect to.`);
+    }
+
+    const envs = await grabComputerEnvs({ computer_id, bot_id });
+    const paths = await writeComputerEnv({ base_url: target.base_url, envs, signal });
+    const secrets = envs.filter((env) => env.secret);
+    const redact = (text: string) => redactSecrets({ text, secrets });
+
+    const client = await connectMcpServer({ url: target.url, signal });
+
+    try {
+        const tools = await grabMcpTools({
+            client,
+            exclude: ComputerMcpExcludedTools,
+            redact,
+            describeImage: ({ data, mime_type }) =>
+                readImage({
+                    url: `data:${mime_type};base64,${data}`,
+                    focus: "This is a screenshot of a computer screen. Describe the page layout, text, buttons and form fields, and any errors or popups.",
+                    signal,
+                    bot_id,
+                }),
+        });
+
+        return {
+            name: target.name,
+            tools: tools.map((tool) => withComputerEnv({ tool, ...paths })),
+            envs: envs.map((env) => ({ key: env.key, description: env.description })),
+            redact,
+            close: () => client.close().catch(() => {}),
+        };
+    } catch (error) {
+        await client.close().catch(() => {});
+        throw error;
+    }
+}