Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 14 Wiki Activity

Add MCP servers and custom tools for bots

Browse Source 
Two new settings pages, MCPs and Tools. Super admins define them; super
admins and admins attach them to bots, to workspaces (every bot there),
or to every bot. Bots get them in chat replies and tasks.

- Custom tools: a JSON-schema of arguments plus an HTTP request, a script
  on the bot's computer, or a prompt, templated with {{args.x}} and
  {{env.KEY}}, secrets redacted from results.
- MCP servers: remote (URL and headers) or stdio commands run on the host
  with a setup command and an isolated env, in a per-process pool. Tools
  are listed into a cache and connect on first call.
- Add, edit, access and try open in modals.
- Markdown, JSON and command fields use twui's AceEditor (CodeEditor),
  its scripts loaded on demand.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-19 15:01:39 +01:00
co-authored by Claude Opus 5.5
parent 550aaeabac
commit c6ec7d77a2
95 changed files with 4733 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/harness/mcp/grab-mcp-dir.ts
+22
View File
@@ -0,0 +1,22 @@
import { mkdir } from "fs/promises";
import path from "path";
import { AppData } from "@/src/data/app-data";
import grabDirNames from "@/src/utils/grab-dir-names";
type Params = {
server_id: number | string;
};
/**
* Where a stdio MCP server runs and its setup installs:
* `DATA_DIR/<AppData.McpDataDir>/<id>`, created when missing. On the data
* volume in production, so installs survive app updates.
*/
export default async function grabMcpDir({ server_id }: Params) {
const { DATA_DIR } = grabDirNames();
const dir = path.join(DATA_DIR, AppData.McpDataDir, String(server_id));
await mkdir(dir, { recursive: true });
return dir;
}
src/harness/mcp/grab-mcp-stdio-env.ts
+23
View File
@@ -0,0 +1,23 @@
import { getDefaultEnvironment } from "@modelcontextprotocol/sdk/client/stdio.js";
import type { McpServerRecord } from "@/src/types";
import fillTemplate from "@/src/utils/fill-template";
type Params = {
server: McpServerRecord;
envs: { key: string; value: string }[];
};
/**
* The only environment a stdio MCP server (and its setup) gets: the
* basics a shell needs (`PATH`, `HOME`, `SHELL`, `USER`, ...) plus its own
* `env`, filled from the envs. Never the app's own environment, which
* holds its encryption key.
*/
export default function grabMcpStdioEnv({ server, envs }: Params): Record<string, string> {
return {
...getDefaultEnvironment(),
...Object.fromEntries(
Object.entries(server.env).map(([key, value]) => [key, fillTemplate({ template: value, envs })]),
),
};
}
src/harness/mcp/grab-mcp-tools.ts
+8 -49
View File
@@ -1,7 +1,8 @@
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { AppData } from "@/src/data/app-data";
import type { HarnessTool } from "../types";
import mcpResultToText, { type McpImage } from "./mcp-result-to-text";
import listMcpTools from "./list-mcp-tools";
import mcpHarnessTool from "./mcp-harness-tool";
import type { McpImage } from "./mcp-result-to-text";
type Params = {
client: Client;
@@ -18,57 +19,15 @@ type Params = {
};
/**
* Every tool an MCP server offers, as harness tools. Arguments go to the
* server as the model wrote them (the server checks them against its own
* schema); results come back as text via `mcpResultToText`.
* Every tool an MCP server offers, as harness tools (`mcpHarnessTool`).
*/
export default async function grabMcpTools({
client,
exclude = [],
describeImage,
redact = (text) => text,
redact,
}: Params): Promise<HarnessTool[]> {
const tools: HarnessTool[] = [];
let cursor: string | undefined;
do {
const page = await client.listTools(cursor ? { cursor } : undefined, {
timeout: AppData.McpConnectTimeoutMs,
});
for (const tool of page.tools) {
if (exclude.includes(tool.name)) continue;
tools.push({
name: tool.name,
description: tool.description || tool.title || tool.name,
parameters: tool.inputSchema as Record<string, unknown>,
run: async ({ args, signal }) => {
try {
return redact(
await mcpResultToText({
result: await client.callTool(
{ name: tool.name, arguments: args },
undefined,
{
signal,
timeout: AppData.McpToolTimeoutMs,
resetTimeoutOnProgress: true,
},
),
describeImage,
}),
);
} catch (error: unknown) {
if (error instanceof Error) error.message = redact(error.message);
throw error;
}
},
});
}
cursor = page.nextCursor;
} while (cursor);
return tools;
return (await listMcpTools({ client }))
.filter((tool) => !exclude.includes(tool.name))
.map((tool) => mcpHarnessTool({ tool, client, describeImage, redact }));
}
src/harness/mcp/host-shell.ts
+7
View File
@@ -0,0 +1,7 @@
/**
* The host's default shell, which runs stdio MCP servers and their setup
* (`<shell> -c <command>`).
*/
export default function hostShell() {
return process.env.SHELL || "/bin/bash";
}
src/harness/mcp/list-mcp-tools.ts
+33
View File
@@ -0,0 +1,33 @@
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { AppData } from "@/src/data/app-data";
import type { McpToolInfo } from "@/src/types";
type Params = {
client: Client;
};
/**
* Every tool an MCP server offers, page by page.
*/
export default async function listMcpTools({ client }: Params): Promise<McpToolInfo[]> {
const tools: McpToolInfo[] = [];
let cursor: string | undefined;
do {
const page = await client.listTools(cursor ? { cursor } : undefined, {
timeout: AppData.McpConnectTimeoutMs,
});
for (const tool of page.tools) {
tools.push({
name: tool.name,
description: tool.description || tool.title || tool.name,
input_schema: tool.inputSchema as Record<string, unknown>,
});
}
cursor = page.nextCursor;
} while (cursor);
return tools;
}
src/harness/mcp/mcp-harness-tool.ts
+59
View File
@@ -0,0 +1,59 @@
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { AppData } from "@/src/data/app-data";
import type { HarnessTool } from "../types";
import mcpResultToText, { type McpImage } from "./mcp-result-to-text";
type Params = {
/**
* What the model calls it. Defaults to the server's name for it.
*/
name?: string;
/**
* The tool as the server lists it.
*/
tool: { name: string; description?: string; input_schema: Record<string, unknown> };
/**
* The connected client, or how to get it on the first call.
*/
client: Client | (() => Promise<Client>);
describeImage?: (image: McpImage) => Promise<string>;
redact?: (text: string) => string;
};
/**
* One MCP server tool as a harness tool. Arguments go to the server as the
* model wrote them (the server checks them against its own schema);
* results come back as text via `mcpResultToText`, redacted.
*/
export default function mcpHarnessTool({
name,
tool,
client,
describeImage,
redact = (text) => text,
}: Params): HarnessTool {
return {
name: name || tool.name,
description: tool.description || tool.name,
parameters: tool.input_schema,
run: async ({ args, signal }) => {
try {
const connected = typeof client == "function" ? await client() : client;
return redact(
await mcpResultToText({
result: await connected.callTool({ name: tool.name, arguments: args }, undefined, {
signal,
timeout: AppData.McpToolTimeoutMs,
resetTimeoutOnProgress: true,
}),
describeImage,
}),
);
} catch (error: unknown) {
if (error instanceof Error) error.message = redact(error.message);
throw error;
}
},
};
}
src/harness/mcp/mcp-stdio-pool.ts
+140
View File
@@ -0,0 +1,140 @@
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
import { AppData } from "@/src/data/app-data";
import { SiteData } from "@/src/data/site-data";
import type { McpServerRecord } from "@/src/types";
import grabMcpDir from "./grab-mcp-dir";
import grabMcpStdioEnv from "./grab-mcp-stdio-env";
import hostShell from "./host-shell";
import runMcpSetup from "./run-mcp-setup";
type Entry = {
/**
* The command, setup and environment it was started with: a change
* restarts it.
*/
key: string;
client: Promise<Client>;
transport?: StdioClientTransport;
timer?: ReturnType<typeof setTimeout>;
};
/**
* The stdio MCP servers running in this process, one per server, shared by
* every run.
*/
const pool = new Map<number, Entry>();
type Params = {
server: McpServerRecord;
envs: { key: string; value: string }[];
};
/**
* A connected client for a stdio MCP server, started on first use (after
* its setup, `runMcpSetup`) with the host's shell in its directory, and
* kept running for later calls until it's idle for `AppData.McpStdioIdleMs`,
* its configuration changes, or it exits. Its stderr's last lines explain
* a failed start.
*/
export default async function grabStdioMcpClient({ server, envs }: Params): Promise<Client> {
const env = grabMcpStdioEnv({ server, envs });
const key = String(Bun.hash(JSON.stringify([server.command, server.setup_command, env])));
const existing = pool.get(server.id);
if (existing?.key == key) {
touch({ server_id: server.id, entry: existing });
return existing.client;
}
if (existing) stopStdioMcpServer({ server_id: server.id });
const entry = { key } as Entry;
entry.client = start({ server, env, entry }).catch((error) => {
if (pool.get(server.id) == entry) pool.delete(server.id);
throw error;
});
pool.set(server.id, entry);
touch({ server_id: server.id, entry });
return entry.client;
}
/**
* Stops a stdio MCP server running in this process, if it is (on a
* change, a delete, or idle).
*/
export function stopStdioMcpServer({ server_id }: { server_id: number }) {
const entry = pool.get(server_id);
if (!entry) return;
pool.delete(server_id);
clearTimeout(entry.timer);
entry.client.then((client) => client.close()).catch(() => {});
}
async function start({ server, env, entry }: { server: McpServerRecord; env: Record<string, string>; entry: Entry }) {
await runMcpSetup({ server, env });
const transport = new StdioClientTransport({
command: hostShell(),
args: ["-c", server.command],
cwd: await grabMcpDir({ server_id: server.id }),
env,
stderr: "pipe",
});
let stderr = "";
transport.stderr?.on("data", (chunk: Buffer) => {
stderr = `${stderr}${chunk.toString()}`.slice(-4000);
});
entry.transport = transport;
const client = new Client({ name: SiteData["SiteSlug"], version: "1.0.0" });
client.onclose = () => {
if (pool.get(server.id) == entry) {
pool.delete(server.id);
clearTimeout(entry.timer);
}
};
try {
await client.connect(transport, { timeout: AppData.McpConnectTimeoutMs * 4 });
} catch (error) {
await client.close().catch(() => {});
throw new Error(
`Couldn't start ${server.name}: ${error instanceof Error ? error.message : String(error)}${
stderr.trim() ? `\n${stderr.trim().slice(-2000)}` : ""
}`,
);
}
return client;
}
function touch({ server_id, entry }: { server_id: number; entry: Entry }) {
clearTimeout(entry.timer);
entry.timer = setTimeout(() => {
if (pool.get(server_id) == entry) stopStdioMcpServer({ server_id });
}, AppData.McpStdioIdleMs);
entry.timer.unref?.();
}
/**
* Children go with the app.
*/
process.once("exit", () => {
for (const entry of pool.values()) {
const pid = entry.transport?.pid;
if (pid) {
try {
process.kill(pid);
} catch {}
}
}
});
src/harness/mcp/open-mcp-server.ts
+48
View File
@@ -0,0 +1,48 @@
import type { Client } from "@modelcontextprotocol/sdk/client/index.js";
import type { McpServerRecord } from "@/src/types";
import fillTemplate from "@/src/utils/fill-template";
import connectMcpServer from "./connect-mcp-server";
import grabStdioMcpClient from "./mcp-stdio-pool";
type Params = {
server: McpServerRecord;
/**
* What its templates (`{{env.KEY}}`) read.
*/
envs: { key: string; value: string }[];
signal?: AbortSignal;
};
/**
* How one run reaches an MCP server, without connecting yet: `client()`
* connects on the first call and is reused after. A remote server gets
* its own connection for the run (closed by `close`); a stdio server is
* shared through the process's pool and keeps running.
*/
export default function openMcpServer({ server, envs, signal }: Params) {
let remote: Promise<Client> | undefined;
const client = () => {
if (server.transport == "stdio") return grabStdioMcpClient({ server, envs });
remote ||= connectMcpServer({
url: fillTemplate({ template: server.url, envs }),
headers: Object.fromEntries(
Object.entries(server.headers).map(([key, value]) => [key, fillTemplate({ template: value, envs })]),
),
signal,
}).catch((error) => {
remote = undefined;
throw error;
});
return remote;
};
const close = async () => {
if (!remote) return;
await (await remote.catch(() => null))?.close().catch(() => {});
};
return { client, close };
}
src/harness/mcp/run-mcp-setup.ts
+64
View File
@@ -0,0 +1,64 @@
import { readdir, rm, writeFile } from "fs/promises";
import path from "path";
import { AppData } from "@/src/data/app-data";
import type { McpServerRecord } from "@/src/types";
import grabMcpDir from "./grab-mcp-dir";
import hostShell from "./host-shell";
type Params = {
server: McpServerRecord;
/**
* Its environment (`grabMcpStdioEnv`).
*/
env: Record<string, string>;
/**
* Runs it even when this setup already ran.
*/
force?: boolean;
};
const MarkerPrefix = ".cobble-setup-";
/**
* Runs a stdio MCP server's setup command with the host's shell in its
* directory, once per setup command: a marker file named after it is left
* behind, so it runs again when the command changes, when `force` is set,
* or when the directory was cleared. The output goes to `setup.log`; a
* failure throws with its last lines.
*/
export default async function runMcpSetup({ server, env, force }: Params) {
const command = server.setup_command.trim();
if (!command) return;
const dir = await grabMcpDir({ server_id: server.id });
const marker = `${MarkerPrefix}${Bun.hash(command).toString(36)}`;
const existing = (await readdir(dir)).filter((file) => file.startsWith(MarkerPrefix));
if (!force && existing.includes(marker)) return;
await Promise.all(existing.map((file) => rm(path.join(dir, file), { force: true })));
const proc = Bun.spawn([hostShell(), "-c", command], {
cwd: dir,
env,
stdout: "pipe",
stderr: "pipe",
timeout: AppData.McpSetupTimeoutMs,
});
const [stdout, stderr, exit_code] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
]);
const output = `${stdout}${stderr ? `\n${stderr}` : ""}`.trim();
await writeFile(path.join(dir, "setup.log"), `$ ${command}\n${output}\nExit code: ${exit_code}\n`);
if (exit_code !== 0) {
throw new Error(`The setup command failed (exit code ${exit_code}): ${output.slice(-2000) || "no output"}`);
}
await writeFile(path.join(dir, marker), command);
}