Add the harness, envs, per-bot models and chat attachments
- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,18 @@
|
||||
type Params = {
|
||||
envs: { key: string; value: string }[];
|
||||
};
|
||||
|
||||
/**
|
||||
* A computer's env files: `shell` for bash (`export KEY='value'`, single
|
||||
* quotes escaped) and `json` for code runners.
|
||||
*/
|
||||
export default function composeComputerEnvFiles({ envs }: Params) {
|
||||
const shell = envs
|
||||
.map((env) => `export ${env.key}='${env.value.replaceAll("'", `'\\''`)}'`)
|
||||
.join("\n");
|
||||
|
||||
return {
|
||||
shell: shell ? `${shell}\n` : "",
|
||||
json: JSON.stringify(Object.fromEntries(envs.map((env) => [env.key, env.value]))),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { ComputerMcpExcludedTools } from "@/src/dict/computer-dict";
|
||||
import grabComputerMcpUrl from "@/src/functions/backend/db/computers/grab-computer-mcp-url";
|
||||
import grabComputerEnvs from "@/src/functions/backend/db/envs/grab-computer-envs";
|
||||
import redactSecrets from "@/src/utils/redact-secrets";
|
||||
import connectMcpServer from "../../mcp/connect-mcp-server";
|
||||
import grabMcpTools from "../../mcp/grab-mcp-tools";
|
||||
import readImage from "../../media/read-image";
|
||||
import type { HarnessTool } from "../../types";
|
||||
import withComputerEnv from "./with-computer-env";
|
||||
import writeComputerEnv from "./write-computer-env";
|
||||
|
||||
type Params = {
|
||||
computer_id: number | string;
|
||||
/**
|
||||
* Screenshots are read with this bot's media reading model, and only
|
||||
* workspaces it shares with the computer add envs.
|
||||
*/
|
||||
bot_id?: number | string;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
export type OpenComputer = {
|
||||
name: string;
|
||||
tools: HarnessTool[];
|
||||
/**
|
||||
* The envs set on the computer, for the prompt: keys and descriptions,
|
||||
* never values.
|
||||
*/
|
||||
envs: { key: string; description: string }[];
|
||||
/**
|
||||
* Blanks the computer's secret values out of text, e.g. the report.
|
||||
*/
|
||||
redact: (text: string) => string;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Connects to a computer (its AIO sandbox, over MCP) and returns its tools:
|
||||
* shell, files, code and browser. Its envs (`grabComputerEnvs`) are written
|
||||
* to the computer and loaded by the shell and code tools; secret values are
|
||||
* redacted from everything the tools return. Screenshots come back as
|
||||
* readings from the media reading harness. Call `close` when the run ends.
|
||||
* Throws when the computer can't be reached, which the queue retries.
|
||||
*/
|
||||
export default async function openComputerTools({
|
||||
computer_id,
|
||||
bot_id,
|
||||
signal,
|
||||
}: Params): Promise<OpenComputer> {
|
||||
const target = await grabComputerMcpUrl({ computer_id });
|
||||
|
||||
if (!target) {
|
||||
throw new Error(`Computer #${computer_id} has no container to connect to.`);
|
||||
}
|
||||
|
||||
const envs = await grabComputerEnvs({ computer_id, bot_id });
|
||||
const paths = await writeComputerEnv({ base_url: target.base_url, envs, signal });
|
||||
const secrets = envs.filter((env) => env.secret);
|
||||
const redact = (text: string) => redactSecrets({ text, secrets });
|
||||
|
||||
const client = await connectMcpServer({ url: target.url, signal });
|
||||
|
||||
try {
|
||||
const tools = await grabMcpTools({
|
||||
client,
|
||||
exclude: ComputerMcpExcludedTools,
|
||||
redact,
|
||||
describeImage: ({ data, mime_type }) =>
|
||||
readImage({
|
||||
url: `data:${mime_type};base64,${data}`,
|
||||
focus: "This is a screenshot of a computer screen. Describe the page layout, text, buttons and form fields, and any errors or popups.",
|
||||
signal,
|
||||
bot_id,
|
||||
}),
|
||||
});
|
||||
|
||||
return {
|
||||
name: target.name,
|
||||
tools: tools.map((tool) => withComputerEnv({ tool, ...paths })),
|
||||
envs: envs.map((env) => ({ key: env.key, description: env.description })),
|
||||
redact,
|
||||
close: () => client.close().catch(() => {}),
|
||||
};
|
||||
} catch (error) {
|
||||
await client.close().catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { HarnessTool } from "../../types";
|
||||
import composeComputerEnvFiles from "./compose-computer-env-files";
|
||||
import withComputerEnv from "./with-computer-env";
|
||||
|
||||
const echoTool = (name: string): HarnessTool => ({
|
||||
name,
|
||||
description: "",
|
||||
parameters: {},
|
||||
run: async ({ args }) => JSON.stringify(args),
|
||||
});
|
||||
|
||||
const paths = { shell_path: "/home/gem/.cobble/env", json_path: "/home/gem/.cobble/env.json" };
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
describe("withComputerEnv", () => {
|
||||
test("sources the env file before bash commands", async () => {
|
||||
const tool = withComputerEnv({ tool: echoTool("sandbox_execute_bash"), ...paths });
|
||||
const sent = JSON.parse(await tool.run({ args: { cmd: "echo $TOKEN" }, signal }));
|
||||
|
||||
expect(sent.cmd).toBe("[ -f /home/gem/.cobble/env ] && . /home/gem/.cobble/env; echo $TOKEN");
|
||||
});
|
||||
|
||||
test("loads the json file before python code", async () => {
|
||||
const tool = withComputerEnv({ tool: echoTool("sandbox_execute_code"), ...paths });
|
||||
const sent = JSON.parse(await tool.run({ args: { code: "print(1)", language: "python" }, signal }));
|
||||
|
||||
expect(sent.code).toStartWith("import json as _cobble_json");
|
||||
expect(sent.code).toEndWith("\nprint(1)");
|
||||
});
|
||||
|
||||
test("leaves other tools alone", () => {
|
||||
const tool = echoTool("browser_navigate");
|
||||
expect(withComputerEnv({ tool, ...paths })).toBe(tool);
|
||||
});
|
||||
});
|
||||
|
||||
describe("composeComputerEnvFiles", () => {
|
||||
test("escapes single quotes for the shell", () => {
|
||||
expect(
|
||||
composeComputerEnvFiles({ envs: [{ key: "A", value: "it's" }] }).shell,
|
||||
).toBe(`export A='it'\\''s'\n`);
|
||||
});
|
||||
|
||||
test("is empty without envs", () => {
|
||||
expect(composeComputerEnvFiles({ envs: [] })).toEqual({ shell: "", json: "{}" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import type { HarnessTool } from "../../types";
|
||||
|
||||
type Params = {
|
||||
tool: HarnessTool;
|
||||
/**
|
||||
* Absolute paths of the env files on the computer.
|
||||
*/
|
||||
shell_path: string;
|
||||
json_path: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Makes a computer tool run with the computer's envs. The sandbox's shells
|
||||
* don't read a profile, so bash commands source the env file first and
|
||||
* Python or JavaScript code loads the JSON one. The model never sees the
|
||||
* addition; other tools pass through.
|
||||
*/
|
||||
export default function withComputerEnv({
|
||||
tool,
|
||||
shell_path,
|
||||
json_path,
|
||||
}: Params): HarnessTool {
|
||||
const shell_prefix = `[ -f ${shell_path} ] && . ${shell_path}; `;
|
||||
|
||||
if (tool.name == "sandbox_execute_bash") {
|
||||
return {
|
||||
...tool,
|
||||
run: ({ args, signal }) =>
|
||||
tool.run({
|
||||
args: typeof args?.cmd == "string" ? { ...args, cmd: `${shell_prefix}${args.cmd}` } : args,
|
||||
signal,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (tool.name == "sandbox_execute_code") {
|
||||
return {
|
||||
...tool,
|
||||
run: ({ args, signal }) =>
|
||||
tool.run({
|
||||
args:
|
||||
typeof args?.code == "string"
|
||||
? { ...args, code: `${codePrefix({ language: args.language, json_path })}${args.code}` }
|
||||
: args,
|
||||
signal,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return tool;
|
||||
}
|
||||
|
||||
function codePrefix({ language, json_path }: { language?: unknown; json_path: string }) {
|
||||
const path = JSON.stringify(json_path);
|
||||
const lang = typeof language == "string" ? language.toLowerCase() : "python";
|
||||
|
||||
if (lang.startsWith("py")) {
|
||||
return `import json as _cobble_json, os as _cobble_os; _cobble_os.environ.update(_cobble_json.load(open(${path})) if _cobble_os.path.exists(${path}) else {})\n`;
|
||||
}
|
||||
|
||||
if (lang.startsWith("js") || lang.startsWith("javascript") || lang.startsWith("node")) {
|
||||
return `try { Object.assign(process.env, JSON.parse(require("fs").readFileSync(${path}, "utf8"))); } catch {}\n`;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import composeComputerEnvFiles from "./compose-computer-env-files";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The sandbox's REST address (no `/mcp`).
|
||||
*/
|
||||
base_url: string;
|
||||
envs: { key: string; value: string }[];
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
/**
|
||||
* Writes the computer's env files (`~/.cobble/env` and `env.json`, owner
|
||||
* only), replacing the last run's, so removed envs disappear too. Returns
|
||||
* their paths for `withComputerEnv`.
|
||||
*/
|
||||
export default async function writeComputerEnv({ base_url, envs, signal }: Params) {
|
||||
const call = async (path: string, init?: RequestInit) => {
|
||||
const res = await fetch(`${base_url}${path}`, {
|
||||
...init,
|
||||
headers: { "content-type": "application/json" },
|
||||
signal: AbortSignal.any([
|
||||
...(signal ? [signal] : []),
|
||||
AbortSignal.timeout(AppData.McpConnectTimeoutMs),
|
||||
]),
|
||||
});
|
||||
const body: any = await res.json().catch(() => ({}));
|
||||
|
||||
if (!res.ok || body?.success === false) {
|
||||
throw new Error(`The computer refused ${path}: ${body?.message || res.status}`);
|
||||
}
|
||||
|
||||
return body;
|
||||
};
|
||||
|
||||
const context = await call(`/v1/sandbox`);
|
||||
const home = String(context.home_dir || "/root").replace(/\/+$/, "");
|
||||
const dir = `${home}/.cobble`;
|
||||
const shell_path = `${dir}/env`;
|
||||
const json_path = `${dir}/env.json`;
|
||||
const files = composeComputerEnvFiles({ envs });
|
||||
|
||||
await call(`/v1/shell/exec`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
command: `mkdir -p ${dir} && chmod 700 ${dir} && : > ${shell_path} && : > ${json_path} && chmod 600 ${shell_path} ${json_path}`,
|
||||
}),
|
||||
});
|
||||
|
||||
for (const [file, content] of [
|
||||
[shell_path, files.shell],
|
||||
[json_path, files.json],
|
||||
] as const) {
|
||||
await call(`/v1/file/write`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ file, content }),
|
||||
});
|
||||
}
|
||||
|
||||
return { shell_path, json_path };
|
||||
}
|
||||
Reference in New Issue
Block a user