Add the harness, envs, per-bot models and chat attachments
- Harness: providers, model catalog synced hourly, automatic free picks, cooldowns, tool loop with context trimming, MCP computer tools - Bots queue runs replies and tasks through the harness; jobs wait (attempt refunded) while every model is resting - Envs: Settings > Environment, scoped general/workspace/computer, written to computers and redacted from tool output - Models: Settings > Models, per-harness provider select defaulting to OpenRouter, and a models modal on each bot's chat (the coordinator's edits the defaults) - Chat: thinking row while a reply runs, file attachments (upload, paste, drop) that replies read as images, audio and document text - Fix media uploads failing after a user's first file, and private files being served from the public media path Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,58 @@
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
|
||||
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import { SiteData } from "@/src/data/site-data";
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* The server's MCP endpoint, e.g. `http://cobble-computer:8080/mcp`.
|
||||
*/
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
/**
|
||||
* Connects to an MCP server over Streamable HTTP, falling back to the
|
||||
* older SSE transport. Close the client when the run is done.
|
||||
*/
|
||||
export default async function connectMcpServer({
|
||||
url,
|
||||
headers,
|
||||
signal,
|
||||
}: Params): Promise<Client> {
|
||||
const endpoint = new URL(url);
|
||||
const request_init = headers ? { headers } : undefined;
|
||||
const options = { signal, timeout: AppData.McpConnectTimeoutMs };
|
||||
|
||||
try {
|
||||
const client = newClient();
|
||||
await client.connect(
|
||||
new StreamableHTTPClientTransport(endpoint, { requestInit: request_init }),
|
||||
options,
|
||||
);
|
||||
return client;
|
||||
} catch (error: unknown) {
|
||||
if (signal?.aborted) throw error;
|
||||
|
||||
try {
|
||||
const client = newClient();
|
||||
await client.connect(
|
||||
new SSEClientTransport(endpoint, { requestInit: request_init }),
|
||||
options,
|
||||
);
|
||||
return client;
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Couldn't connect to the MCP server at ${endpoint.origin}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function newClient() {
|
||||
return new Client({ name: SiteData["SiteSlug"], version: "1.0.0" });
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import type { HarnessTool } from "../types";
|
||||
import mcpResultToText, { type McpImage } from "./mcp-result-to-text";
|
||||
|
||||
type Params = {
|
||||
client: Client;
|
||||
/**
|
||||
* Tool names to leave out.
|
||||
*/
|
||||
exclude?: readonly string[];
|
||||
describeImage?: (image: McpImage) => Promise<string>;
|
||||
/**
|
||||
* Applied to every result (and error) before the model sees it, e.g.
|
||||
* to hide secret values.
|
||||
*/
|
||||
redact?: (text: string) => string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Every tool an MCP server offers, as harness tools. Arguments go to the
|
||||
* server as the model wrote them (the server checks them against its own
|
||||
* schema); results come back as text via `mcpResultToText`.
|
||||
*/
|
||||
export default async function grabMcpTools({
|
||||
client,
|
||||
exclude = [],
|
||||
describeImage,
|
||||
redact = (text) => text,
|
||||
}: Params): Promise<HarnessTool[]> {
|
||||
const tools: HarnessTool[] = [];
|
||||
let cursor: string | undefined;
|
||||
|
||||
do {
|
||||
const page = await client.listTools(cursor ? { cursor } : undefined, {
|
||||
timeout: AppData.McpConnectTimeoutMs,
|
||||
});
|
||||
|
||||
for (const tool of page.tools) {
|
||||
if (exclude.includes(tool.name)) continue;
|
||||
|
||||
tools.push({
|
||||
name: tool.name,
|
||||
description: tool.description || tool.title || tool.name,
|
||||
parameters: tool.inputSchema as Record<string, unknown>,
|
||||
run: async ({ args, signal }) => {
|
||||
try {
|
||||
return redact(
|
||||
await mcpResultToText({
|
||||
result: await client.callTool(
|
||||
{ name: tool.name, arguments: args },
|
||||
undefined,
|
||||
{
|
||||
signal,
|
||||
timeout: AppData.McpToolTimeoutMs,
|
||||
resetTimeoutOnProgress: true,
|
||||
},
|
||||
),
|
||||
describeImage,
|
||||
}),
|
||||
);
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof Error) error.message = redact(error.message);
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
cursor = page.nextCursor;
|
||||
} while (cursor);
|
||||
|
||||
return tools;
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import mcpResultToText from "./mcp-result-to-text";
|
||||
|
||||
describe("mcpResultToText", () => {
|
||||
test("joins text parts", async () => {
|
||||
expect(
|
||||
await mcpResultToText({
|
||||
result: {
|
||||
content: [
|
||||
{ type: "text", text: "one" },
|
||||
{ type: "text", text: "two" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
).toBe("one\n\ntwo");
|
||||
});
|
||||
|
||||
test("describes images with the given reader", async () => {
|
||||
expect(
|
||||
await mcpResultToText({
|
||||
result: {
|
||||
content: [{ type: "image", data: "aGk=", mimeType: "image/png" }],
|
||||
},
|
||||
describeImage: async ({ data, mime_type }) =>
|
||||
`${mime_type} ${data}`,
|
||||
}),
|
||||
).toBe("[Image: image/png aGk=]");
|
||||
});
|
||||
|
||||
test("notes images when nothing can read them", async () => {
|
||||
expect(
|
||||
await mcpResultToText({
|
||||
result: {
|
||||
content: [{ type: "image", data: "aGk=", mimeType: "image/png" }],
|
||||
},
|
||||
}),
|
||||
).toBe("[Image (image/png)]");
|
||||
});
|
||||
|
||||
test("keeps going when an image can't be read", async () => {
|
||||
expect(
|
||||
await mcpResultToText({
|
||||
result: { content: [{ type: "image", data: "aGk=" }] },
|
||||
describeImage: async () => {
|
||||
throw new Error("No vision model.");
|
||||
},
|
||||
}),
|
||||
).toBe("[Image: couldn't be read. No vision model.]");
|
||||
});
|
||||
|
||||
test("falls back to structured content", async () => {
|
||||
expect(
|
||||
await mcpResultToText({
|
||||
result: { content: [], structuredContent: { ok: true } },
|
||||
}),
|
||||
).toBe(`{"ok":true}`);
|
||||
});
|
||||
|
||||
test("throws error results", async () => {
|
||||
expect(
|
||||
mcpResultToText({
|
||||
result: {
|
||||
isError: true,
|
||||
content: [{ type: "text", text: "No such file" }],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("No such file");
|
||||
});
|
||||
|
||||
test("says when the tool returned nothing", async () => {
|
||||
expect(await mcpResultToText({ result: {} })).toBe(
|
||||
"Done. The tool returned nothing.",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
export type McpImage = {
|
||||
/**
|
||||
* Base64.
|
||||
*/
|
||||
data: string;
|
||||
mime_type: string;
|
||||
};
|
||||
|
||||
type Params = {
|
||||
/**
|
||||
* A `tools/call` result.
|
||||
*/
|
||||
result: {
|
||||
content?: unknown;
|
||||
structuredContent?: unknown;
|
||||
isError?: unknown;
|
||||
/**
|
||||
* Old servers (protocol 2024-10-07) answer with this instead.
|
||||
*/
|
||||
toolResult?: unknown;
|
||||
};
|
||||
/**
|
||||
* Turns an image (a screenshot) into text for the model. Without it,
|
||||
* images are noted but not described.
|
||||
*/
|
||||
describeImage?: (image: McpImage) => Promise<string>;
|
||||
};
|
||||
|
||||
/**
|
||||
* An MCP tool result as text for the model: text as is, images described,
|
||||
* resources by their text or link. Tool messages carry text only, so an
|
||||
* image reaches the model as its reading. A result flagged as an error is
|
||||
* thrown, so the model gets it as `Error: ...`.
|
||||
*/
|
||||
export default async function mcpResultToText({
|
||||
result,
|
||||
describeImage,
|
||||
}: Params): Promise<string> {
|
||||
const parts = Array.isArray(result.content) ? result.content : [];
|
||||
const texts: string[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
texts.push(await partToText({ part, describeImage }));
|
||||
}
|
||||
|
||||
const structured = result.structuredContent ?? result.toolResult;
|
||||
|
||||
if (!texts.length && structured !== undefined) {
|
||||
texts.push(typeof structured == "string" ? structured : JSON.stringify(structured));
|
||||
}
|
||||
|
||||
const text = texts.filter(Boolean).join("\n\n") || "Done. The tool returned nothing.";
|
||||
|
||||
if (result.isError) {
|
||||
throw new Error(text);
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async function partToText({
|
||||
part,
|
||||
describeImage,
|
||||
}: {
|
||||
part: any;
|
||||
describeImage?: Params["describeImage"];
|
||||
}): Promise<string> {
|
||||
switch (part?.type) {
|
||||
case "text":
|
||||
return String(part.text ?? "");
|
||||
|
||||
case "image": {
|
||||
if (!describeImage || typeof part.data != "string") {
|
||||
return `[Image (${part.mimeType || "unknown type"})]`;
|
||||
}
|
||||
|
||||
try {
|
||||
return `[Image: ${await describeImage({
|
||||
data: part.data,
|
||||
mime_type: part.mimeType || "image/png",
|
||||
})}]`;
|
||||
} catch (error: unknown) {
|
||||
return `[Image: couldn't be read. ${error instanceof Error ? error.message : String(error)}]`;
|
||||
}
|
||||
}
|
||||
|
||||
case "audio":
|
||||
return `[Audio (${part.mimeType || "unknown type"})]`;
|
||||
|
||||
case "resource":
|
||||
return typeof part.resource?.text == "string"
|
||||
? part.resource.text
|
||||
: `[File: ${part.resource?.uri || "unnamed"}]`;
|
||||
|
||||
case "resource_link":
|
||||
return `[Link: ${part.name || part.uri} ${part.uri || ""}]`.trim();
|
||||
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user