Add MCP servers and custom tools for bots
Two new settings pages, MCPs and Tools. Super admins define them; super
admins and admins attach them to bots, to workspaces (every bot there),
or to every bot. Bots get them in chat replies and tasks.
- Custom tools: a JSON-schema of arguments plus an HTTP request, a script
on the bot's computer, or a prompt, templated with {{args.x}} and
{{env.KEY}}, secrets redacted from results.
- MCP servers: remote (URL and headers) or stdio commands run on the host
with a setup command and an isolated env, in a per-process pool. Tools
are listed into a cache and connect on first call.
- Add, edit, access and try open in modals.
- Markdown, JSON and command fields use twui's AceEditor (CodeEditor),
its scripts loaded on demand.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,39 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { beforeAll, describe, expect, test } from "bun:test";
|
||||
import botExtensionsSql from "./bot-extensions-sql";
|
||||
|
||||
const db = new Database(":memory:");
|
||||
|
||||
beforeAll(() => {
|
||||
db.run(`CREATE TABLE custom_tools (id INTEGER PRIMARY KEY, all_bots INTEGER, enabled INTEGER)`);
|
||||
db.run(`CREATE TABLE bot_custom_tools (bot_id INTEGER, custom_tool_id INTEGER)`);
|
||||
db.run(`CREATE TABLE workspace_custom_tools (workspace_id INTEGER, custom_tool_id INTEGER)`);
|
||||
/**
|
||||
* 1: every bot. 2: bot 5. 3: workspace 9. 4: every bot, off.
|
||||
* 5: nobody.
|
||||
*/
|
||||
db.run(`INSERT INTO custom_tools VALUES (1, 1, 1), (2, 0, 1), (3, 0, 1), (4, 1, 0), (5, 0, 1)`);
|
||||
db.run(`INSERT INTO bot_custom_tools VALUES (5, 2)`);
|
||||
db.run(`INSERT INTO workspace_custom_tools VALUES (9, 3)`);
|
||||
});
|
||||
|
||||
function ids(params: Parameters<typeof botExtensionsSql>[0]) {
|
||||
const { sql, values } = botExtensionsSql(params);
|
||||
return db
|
||||
.query(sql)
|
||||
.all(...values)
|
||||
.map((row) => (row as { id: number }).id);
|
||||
}
|
||||
|
||||
describe("botExtensionsSql", () => {
|
||||
test("a bot gets every-bot ones and its own, never disabled ones", () => {
|
||||
expect(ids({ kind: "tool", bot_id: 5 })).toEqual([1, 2]);
|
||||
expect(ids({ kind: "tool", bot_id: 6 })).toEqual([1]);
|
||||
});
|
||||
|
||||
test("in a workspace, the workspace's too", () => {
|
||||
expect(ids({ kind: "tool", bot_id: 6, workspace_id: 9 })).toEqual([1, 3]);
|
||||
expect(ids({ kind: "tool", bot_id: 5, workspace_id: 9 })).toEqual([1, 2, 3]);
|
||||
expect(ids({ kind: "tool", bot_id: 5, workspace_id: 8 })).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
type Params = {
|
||||
/**
|
||||
* Which kind: MCP servers or custom tools.
|
||||
*/
|
||||
kind: "mcp" | "tool";
|
||||
bot_id: number | string;
|
||||
/**
|
||||
* The workspace the run is in: what's attached to it counts too.
|
||||
*/
|
||||
workspace_id?: number | string | null;
|
||||
};
|
||||
|
||||
const Tables = {
|
||||
mcp: {
|
||||
table: "mcp_servers",
|
||||
bot_link: "bot_mcp_servers",
|
||||
workspace_link: "workspace_mcp_servers",
|
||||
column: "mcp_server_id",
|
||||
},
|
||||
tool: {
|
||||
table: "custom_tools",
|
||||
bot_link: "bot_custom_tools",
|
||||
workspace_link: "workspace_custom_tools",
|
||||
column: "custom_tool_id",
|
||||
},
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* The enabled MCP servers or custom tools a bot gets on one run: those
|
||||
* for every bot, those attached to the bot, and in a workspace those
|
||||
* attached to the workspace.
|
||||
*/
|
||||
export default function botExtensionsSql({ kind, bot_id, workspace_id }: Params) {
|
||||
const { table, bot_link, workspace_link, column } = Tables[kind];
|
||||
|
||||
return {
|
||||
sql: `SELECT x.* FROM ${table} x WHERE x.enabled = 1 AND (
|
||||
x.all_bots = 1
|
||||
OR x.id IN (SELECT ${column} FROM ${bot_link} WHERE bot_id = ?)
|
||||
${workspace_id ? `OR x.id IN (SELECT ${column} FROM ${workspace_link} WHERE workspace_id = ?)` : ""}
|
||||
) ORDER BY x.id ASC`,
|
||||
values: workspace_id ? [Number(bot_id), Number(workspace_id)] : [Number(bot_id)],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { ReservedToolNames, ReservedToolPrefixes } from "@/src/dict/custom-tool-dict";
|
||||
|
||||
type Params = {
|
||||
name: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Why a custom tool can't take this name, or null when it can: snake_case
|
||||
* of 2 to 64 characters, never a built-in tool's name or a computer tool's
|
||||
* prefix, and never with `__` (MCP tools use it, `<server>__<tool>`).
|
||||
*/
|
||||
export default function checkCustomToolName({ name }: Params): string | null {
|
||||
if (!/^[a-z][a-z0-9_]{1,63}$/.test(name)) {
|
||||
return `Use lowercase letters, numbers and underscores, starting with a letter (2 to 64 characters).`;
|
||||
}
|
||||
|
||||
if (name.includes("__")) return `Don't use a double underscore: MCP tools use it.`;
|
||||
|
||||
if ((ReservedToolNames as readonly string[]).includes(name)) {
|
||||
return `${name} is a built-in tool's name.`;
|
||||
}
|
||||
|
||||
const prefix = ReservedToolPrefixes.find((p) => name.startsWith(p));
|
||||
|
||||
if (prefix) return `Names starting with ${prefix} are kept for computers' tools.`;
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import checkCustomToolName from "./check-custom-tool-name";
|
||||
import mcpServerSlug from "./mcp-server-slug";
|
||||
import mcpToolName from "./mcp-tool-name";
|
||||
|
||||
describe("checkCustomToolName", () => {
|
||||
test("takes snake_case", () => {
|
||||
expect(checkCustomToolName({ name: "get_weather" })).toBeNull();
|
||||
});
|
||||
|
||||
test("refuses built-ins, computer prefixes, double underscores and bad shapes", () => {
|
||||
expect(checkCustomToolName({ name: "start_task" })).toContain("built-in");
|
||||
expect(checkCustomToolName({ name: "sandbox_run" })).toContain("sandbox_");
|
||||
expect(checkCustomToolName({ name: "a__b" })).toContain("double underscore");
|
||||
expect(checkCustomToolName({ name: "Get-Weather" })).toContain("lowercase");
|
||||
expect(checkCustomToolName({ name: "1abc" })).toContain("lowercase");
|
||||
});
|
||||
});
|
||||
|
||||
describe("mcpToolName", () => {
|
||||
test("prefixes with the slug, cleans and caps the name", () => {
|
||||
expect(mcpToolName({ slug: "github", name: "create.issue" })).toBe("github__create_issue");
|
||||
expect(mcpToolName({ slug: "s", name: "x".repeat(80) })).toHaveLength(64);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mcpServerSlug", () => {
|
||||
test("slugs a name", () => {
|
||||
expect(mcpServerSlug({ name: "GitHub (work)" })).toBe("github_work");
|
||||
expect(mcpServerSlug({ name: "!!!" })).toBe("mcp");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import fillTemplate from "./fill-template";
|
||||
|
||||
describe("fillTemplate", () => {
|
||||
test("fills args and envs", () => {
|
||||
expect(
|
||||
fillTemplate({
|
||||
template: "https://api.test/{{args.city}}?key={{ env.API_KEY }}",
|
||||
args: { city: "Lagos" },
|
||||
envs: [{ key: "API_KEY", value: "abc" }],
|
||||
}),
|
||||
).toBe("https://api.test/Lagos?key=abc");
|
||||
});
|
||||
|
||||
test("objects become JSON, missing args empty, dotted paths reach in", () => {
|
||||
expect(
|
||||
fillTemplate({
|
||||
template: "{{args.filter}}|{{args.missing}}|{{args.filter.tag}}",
|
||||
args: { filter: { tag: "x" } },
|
||||
}),
|
||||
).toBe('{"tag":"x"}||x');
|
||||
});
|
||||
|
||||
test("encodes filled arguments only", () => {
|
||||
expect(
|
||||
fillTemplate({
|
||||
template: "{{env.BASE}}/q?s={{args.q}}&x=a b",
|
||||
args: { q: "a b&c" },
|
||||
envs: [{ key: "BASE", value: "https://x.test/v1" }],
|
||||
encode: encodeURIComponent,
|
||||
}),
|
||||
).toBe("https://x.test/v1/q?s=a%20b%26c&x=a b");
|
||||
});
|
||||
|
||||
test("an unset env throws, naming it", () => {
|
||||
expect(() => fillTemplate({ template: "{{env.NOPE}}" })).toThrow("NOPE");
|
||||
});
|
||||
|
||||
test("leaves other braces alone", () => {
|
||||
expect(fillTemplate({ template: '{"a": "{{other}}"}' })).toBe('{"a": "{{other}}"}');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
type Params = {
|
||||
template: string;
|
||||
/**
|
||||
* `{{args.name}}`, dotted paths reach into objects. Missing ones fill
|
||||
* in empty; objects and arrays as JSON.
|
||||
*/
|
||||
args?: Record<string, unknown>;
|
||||
/**
|
||||
* `{{env.KEY}}`. A key that isn't set throws, naming it.
|
||||
*/
|
||||
envs?: { key: string; value: string }[];
|
||||
/**
|
||||
* Applied to every filled-in argument, e.g. `encodeURIComponent` in a
|
||||
* URL. Envs go in as they are (one may hold a whole base URL).
|
||||
*/
|
||||
encode?: (value: string) => string;
|
||||
};
|
||||
|
||||
const Pattern = /\{\{\s*(args|env)\.([A-Za-z0-9_.-]+)\s*\}\}/g;
|
||||
|
||||
/**
|
||||
* Fills a custom tool's or MCP server's template: `{{args.x}}` from the
|
||||
* model's arguments, `{{env.KEY}}` from the envs. Anything else is left as
|
||||
* written.
|
||||
*/
|
||||
export default function fillTemplate({ template, args = {}, envs = [], encode = (v) => v }: Params) {
|
||||
return template.replace(Pattern, (_match, source: string, path: string) => {
|
||||
if (source == "env") {
|
||||
const env = envs.find((e) => e.key == path);
|
||||
|
||||
if (!env) throw new Error(`The env ${path} isn't set. Add it in Settings, under Environment.`);
|
||||
|
||||
return env.value;
|
||||
}
|
||||
|
||||
return encode(stringify({ value: pick({ value: args, path }) }));
|
||||
});
|
||||
}
|
||||
|
||||
function pick({ value, path }: { value: unknown; path: string }) {
|
||||
return path.split(".").reduce<unknown>((current, key) => {
|
||||
if (current && typeof current == "object") return (current as Record<string, unknown>)[key];
|
||||
return undefined;
|
||||
}, value);
|
||||
}
|
||||
|
||||
function stringify({ value }: { value: unknown }) {
|
||||
if (value === undefined || value === null) return "";
|
||||
if (typeof value == "string") return value;
|
||||
if (typeof value == "number" || typeof value == "boolean") return String(value);
|
||||
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
const AceScripts = [
|
||||
"https://cdnjs.cloudflare.com/ajax/libs/ace/1.22.0/ace.min.js",
|
||||
"https://cdnjs.cloudflare.com/ajax/libs/ace/1.22.0/ext-language_tools.min.js",
|
||||
];
|
||||
|
||||
let loading: Promise<void> | undefined;
|
||||
|
||||
/**
|
||||
* Loads the ACE editor's scripts once per page, in order (the language
|
||||
* tools need ACE itself), so only pages with an editor download them.
|
||||
* twui's `AceEditor` waits for `window.ace` on its own.
|
||||
*/
|
||||
export default function loadAceScripts() {
|
||||
if (typeof window == "undefined") return Promise.resolve();
|
||||
|
||||
loading ||= AceScripts.reduce(
|
||||
(previous, src) =>
|
||||
previous.then(
|
||||
() =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
if (document.querySelector(`script[src="${src}"]`)) return resolve();
|
||||
|
||||
const script = document.createElement("script");
|
||||
script.src = src;
|
||||
script.onload = () => resolve();
|
||||
script.onerror = () => reject(new Error(`Couldn't load ${src}`));
|
||||
document.head.appendChild(script);
|
||||
}),
|
||||
),
|
||||
Promise.resolve(),
|
||||
).catch((error) => {
|
||||
loading = undefined;
|
||||
throw error;
|
||||
});
|
||||
|
||||
return loading;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
type Params = {
|
||||
name: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A server's slug from its name: lowercase letters, numbers and single
|
||||
* underscores, at most 24 characters, so its tools' names stay short.
|
||||
*/
|
||||
export default function mcpServerSlug({ name }: Params) {
|
||||
const slug = name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "_")
|
||||
.replace(/^_+|_+$/g, "")
|
||||
.slice(0, 24)
|
||||
.replace(/_+$/g, "");
|
||||
|
||||
return slug || "mcp";
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
type Params = {
|
||||
/**
|
||||
* The server's slug (`mcp_servers.slug`).
|
||||
*/
|
||||
slug: string;
|
||||
/**
|
||||
* The tool's name on the server.
|
||||
*/
|
||||
name: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* What a bot calls an MCP server's tool: `<slug>__<tool>`, only letters,
|
||||
* numbers, `_` and `-`, at most 64 characters (what providers accept).
|
||||
*/
|
||||
export default function mcpToolName({ slug, name }: Params) {
|
||||
return `${slug}__${name.replace(/[^A-Za-z0-9_-]/g, "_")}`.slice(0, 64);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import parseTemplateLines, { templateLines } from "./parse-template-lines";
|
||||
|
||||
describe("parseTemplateLines", () => {
|
||||
test("reads headers, keeping colons in values", () => {
|
||||
expect(
|
||||
parseTemplateLines({ text: "Authorization: Bearer {{env.T}}\n\nX-Url: http://a:1\nbad line", separator: ":" }),
|
||||
).toEqual({ Authorization: "Bearer {{env.T}}", "X-Url": "http://a:1" });
|
||||
});
|
||||
|
||||
test("reads envs and writes them back", () => {
|
||||
const map = parseTemplateLines({ text: "A=1\nB = x=y", separator: "=" });
|
||||
expect(map).toEqual({ A: "1", B: "x=y" });
|
||||
expect(templateLines({ map, separator: "=" })).toBe("A=1\nB=x=y");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
type Params = {
|
||||
text: string;
|
||||
/**
|
||||
* What separates a name from its value: `:` for headers, `=` for envs.
|
||||
*/
|
||||
separator: ":" | "=";
|
||||
};
|
||||
|
||||
/**
|
||||
* Reads one `Name: value` (or `NAME=value`) per line into a map, for the
|
||||
* header and env fields of an MCP server or tool. Blank lines and lines
|
||||
* without the separator are skipped; the value is everything after the
|
||||
* first one, trimmed.
|
||||
*/
|
||||
export default function parseTemplateLines({ text, separator }: Params): Record<string, string> {
|
||||
const map: Record<string, string> = {};
|
||||
|
||||
for (const line of text.split("\n")) {
|
||||
const at = line.indexOf(separator);
|
||||
const name = at > 0 ? line.slice(0, at).trim() : "";
|
||||
|
||||
if (name) map[name] = line.slice(at + 1).trim();
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* The other way: a map as one `Name: value` (or `NAME=value`) per line.
|
||||
*/
|
||||
export function templateLines({ map, separator }: { map?: Record<string, string>; separator: ":" | "=" }) {
|
||||
return Object.entries(map || {})
|
||||
.map(([name, value]) => `${name}${separator == ":" ? ": " : "="}${value}`)
|
||||
.join("\n");
|
||||
}
|
||||
Reference in New Issue
Block a user