diff --git a/AGENTS.md b/AGENTS.md index 120fdcb..36036c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,6 +132,9 @@ Envs are set in Settings, under Environment (super admin only), not in `.env`. T - **Per-bot models:** in a workspace, its `workspace_harness_settings` row wins (`source: "workspace"`). Otherwise a worker bot inherits the defaults per harness unless it has a `bot_harness_settings` row (same fields as `harness_settings`, unique per bot and harness). `grabHarnessSetting({ harness, bot_id })` returns the bot's row (`source: "bot"`) or the default; the coordinator always gets the default. Every entry point takes `bot_id` and passes it on (text loop -> media reading/transcription), so a bot's run uses its own models throughout. Saved by `saveBotHarnessSetting` (`inherit` deletes the row); the page data is `grabBotHarnessOverview`. A bot's chat header has a ghost model icon (`(partials)/header-icon-button.tsx`; the text model and provider show under the bot's name) that opens a modal with every harness (`(partials)/bot-models-modal.tsx`); the modal and `/admin/bots/[bot_id]/models` both use `(partials)/bot-harness-form.tsx`. On the coordinator the modal edits the defaults themselves (`api/admin/harness/save-settings`, the same settings as Settings, under Models, without the providers): super admins only, the icon is disabled for everyone else. Every one of these wraps `settings/models/(partials)/harness-settings-form.tsx`: per harness a provider select first (`harness-provider-select.tsx`: any provider, one provider, or on a bot "Same as the default"), then the model select narrowed to it. Removing a provider (`deleteHarnessProvider`) puts default and bot settings that picked it or its models back on any provider (`save` prop; "Same as the default" shows when the overview item has `default`). - **Tools:** `HarnessTool` (built per run with its bot bound): own tools validate with a zod `schema`; tools described elsewhere pass a JSON schema as `parameters` and validate themselves. `run-tool-call.ts` reports bad calls and failures back to the model as `Error: ...`. Bot tools in `src/harness/tools/bot/`: `update_memory`, `update_instructions` (`grabBotTools`, every run), `start_task` and, for the coordinator with workers, `delegate_task` (`grabBotChatTools`, chat replies only; the job's `requested_by_bot_id` is the coordinator, and `relayJobToRequester` posts the worker's result or failure in its conversation, pushed as `relay` by `pushBotNotice`), `ask_user_to_take_over` (see Take-over), plus for the coordinator `create_workspace` and `update_workspace` (a new workspace with its shared context, or changing one; both also in its tasks, so a task that gathers material can write it into the workspace), `create_bot` (add a worker with its name, role and instructions, and join it to a workspace straight away) and, in its own chat, `add_bot_to_workspace` (put an existing worker in one of the user's workspaces) and `assign_computer` (grant or revoke a worker's direct access, `add/remove-bot-to/from-computer`; access through a workspace stays and the result says so). `search_computers` (`src/harness/tools/computer/search-computers-tool.ts`, chat replies and tasks) runs one read-only command on all of a bot's computers at once over the sandbox REST API (`run-computer-command.ts`), each output under its `[computer:ID]`, secrets redacted, capped by `AppData.ComputerSearch*`. - **MCP:** `src/harness/mcp/`: `connectMcpServer` (Streamable HTTP, SSE fallback), `grabMcpTools` (each server tool becomes a `HarnessTool` with its JSON schema), `mcpResultToText` (text as is, images read by the vision harness, `isError` thrown). Computers are reached this way: `openComputerTools` (`src/harness/tools/computer/`) connects to the computer's AIO sandbox at `grabComputerMcpUrl` (the `MAIN_COMPUTER_URL` app setting for the main computer, for a dev server outside docker, e.g. `http://localhost:8170`, else `http://:/mcp`, or its IP address when the hostname can't be reached), leaves out `ComputerMcpExcludedTools`, and sets the computer's envs (see Envs). +- **MCPs and custom tools (extensions):** installs add their own MCP servers (`mcp_servers`, Settings, under MCPs) and tools (`custom_tools`, Settings, under Tools). Each goes to the bots it's attached to (`bot_mcp_servers`, `bot_custom_tools`), to every bot in a workspace there (`workspace_mcp_servers`, `workspace_custom_tools`), or to every bot (`all_bots`, the coordinator included); `botExtensionsSql` (`src/utils/bot-extensions-sql.ts`, tested) resolves a run's, `grabBotExtensions` reads them. Super admins define them; super admins and admins attach them (`setExtensionAccess`, an admin never touches the coordinator's or `all_bots`). Helpers in `src/functions/backend/db/extensions/`, API in `api/admin/mcps/` and `api/admin/tools/`, zod in `src/schemas/extension-schema.ts`. `openBotExtensions` (`src/harness/tools/extensions/`) turns them into harness tools for chat replies and tasks (built-ins win a name clash; `close()` in the runners' `finally`; `note` tells the bot about a server it can't use). Templates (`fillTemplate`, `src/utils/fill-template.ts`): `{{args.x}}` and `{{env.KEY}}`, envs from `grabRunEnvs` (workspace beats general, app settings never, no need to be set on computers); secret values are redacted from every result. + - **Custom tools** (`CustomToolKinds` in `src/dict/custom-tool-dict.ts`): a name (`checkCustomToolName`: snake_case, never a built-in's, `ReservedToolNames`), a description, a JSON-schema `parameters` and a `config` per kind, built by `buildCustomTool` (`src/harness/tools/custom/`): `http` (`runHttpTool`: the app fetches, arguments URL-encoded in the URL, no body template sends the arguments as JSON), `script` (`runScriptTool`: bash/python/javascript on the run's computer, the task's or in chat the bot's current one, over the sandbox REST API in a subshell, computer envs set, arguments as JSON in `COBBLE_ARGS` and on stdin; left out without a computer), `prompt` (`runPromptTool`: one `generateText` turn with the bot's models). Super admins try one from the page (`testCustomTool`, the main computer for scripts). + - **MCP servers** (`McpTransports` in `src/dict/mcp-dict.ts`): `remote` (URL plus templated headers, `connectMcpServer`) or `stdio` (a command run with the host's shell, `hostShell`, in `DATA_DIR/mcp/` (`grabMcpDir`), after its `setup_command`, which `runMcpSetup` reruns whenever it changes (marker file) or on Reinstall; its env is only its own `env` map plus the shell basics, `grabMcpStdioEnv`, never the app's). Their tools are listed into `tools_cache` (`syncMcpServer`: on save, Refresh, and hourly for remote ones by the `sync-mcp-servers` cron) and named `__` (`mcpToolName`); runs build tools from the cache (`mcpHarnessTool`) and connect on the first call (`openMcpServer`). stdio servers live in a per-process pool (`src/harness/mcp/mcp-stdio-pool.ts`), stopped after `AppData.McpStdioIdleMs` idle, on save or delete. - **Replies and tasks:** `run-bot-reply.ts` answers from the last `BotReplyHistoryMessages` messages (`botMessagesToHarness`) with `ChatReplyNote`, the bot's computers (`compose-computers-note.ts`: what they are and that they're reached through `start_task`) and the chat tools, at most `BotReplyMaxToolSteps` steps, and finishes silently if the bot already had the last word (`enqueueBotReply` reuses a pending reply, so a burst of messages gets one). `run-bot-task.ts` runs the brief (`compose-job-brief.ts`) with `TaskRunNote`, the bot tools and the job's computer tools, closing the MCP connection after. Retryable harness errors go back to the queue's backoff, the rest are `BotJobFatalError`. - **API:** `api/admin/harness/` (`save-provider` also syncs, `sync-provider`, `delete-provider`, `save-settings`), super admin only, each returning the fresh `grabHarnessOverview()` page data. @@ -146,7 +149,7 @@ Live routes under `/admin` (see `AdminSections`): `/admin/bots`, `/admin/workspa - `/admin/help` (live) - familiar problems and their fixes, one `SettingsGroup` per topic (`(data)/help-topics.ts`, markdown rendered with the chat's `MessageMarkdownComponents`, the topic id as the group's anchor). First topic: signing in to Google and other strict sites with Cookie-Editor over the Desktop view (`HelpSignInTopicId` in `src/dict/help-dict.ts`; `ask_user_to_take_over` points bots to it) - Routines, Activity and Help are the `RailPages` (`src/dict/rail-pages-dict.ts`), opened from the rail's user popover (which lists them, then Settings and Sign out). On them the rail shows its own aside (`(partials)/rail-pages-list.tsx`): a back link to the bots, then those pages, like settings - `/admin/backups` - super admin only: create/list/download database snapshots in `db/.backups`, plus `restore/` (restore a listed backup, or upload a SQLite database file which is saved to `.backups` then restored). Shells out to the `bun-sqlite` CLI (`backup`, and `import ` since `restore` is interactive) via `src/functions/backend/backups/`; API routes in `api/admin/backups/` -- `/admin/settings` (live) - redirects to `SettingsDefaultPath` (`/admin/settings/general`). Sub pages: `general/` (theme: System/Light/Dark per device via `useThemePreference`, stored in `localStorage[AppData.ThemeStorageKey]`, the same key `ThemeInitScript` and twui's `ColorSchemeSelector` read; applies instantly, no save bar), `profile/` (current user profile), `change-password/` (set/change own password) `models/` (super admin only, `super_admin_only` in the dict hides it from the rail: model providers and which model each harness uses) and `environment/` (super admin only: app settings and the envs bots and computers use, per scope), each wrapped in `SettingsShell` (`(partials)/settings-shell.tsx`: 64px pane header like the conversation header, then one centered column). Content is built from `SettingsGroup` blocks (label + description left, controls right, hairline-separated, no cards) and ends with the sticky `SettingsSaveBar` (Save enabled only when dirty, Discard, one-line status). Saves don't reload the page. Keep save notices in component state: `useFormInit` clears `status` whenever the form changes. `bots/` (the crew: chat, edit with `BotForm`, models, delete a worker via `api/admin/bots/delete-bot`), `workspaces/` (open, create through the Add flow, delete), `computers/` (state and resources, live; super admins create, edit with `AddComputerForm`, retry, delete), those three for `canManageCrew` only (`grab-crew-settings-server-props.ts`, `can_view` in the dict) and `users/` (for `canListUsers`: super admins and admins; the dict's `can_view` gates it in the rail; add with a password (`canCreateUser`), edit (`canUpdateUser`), both with the one `UserForm` (`admin/people/create/(partials)/user-form/`, `useFormInit`, photo included, saved in one request), revoke (adds `revoked`, `canRevokeUser`), delete (`canRevokeUser`'s rules, never yourself), pick their user types from `UserTypes` (`user-types-select.tsx`: only the ones `filterAssignableUserTypes` allows, never `UneditableUserTypes`; at least one, since login needs a type; `saveUser` checks each grant and removal with `userTypeAssignment`; the super admin's types are never shown or changed there, and they can't be revoked; the password is entered twice, `SaveUserRequestSchema` checks they match), delete with everything theirs; `api/admin/users/save-user`, `delete-user`). The lists share `SettingsList`, `SettingsListRow`, `SettingsRowAction` and `use-settings-list`. Sub pages are declared once in `src/dict/settings-pages-dict.ts`; on every `/admin/settings/*` page the rail swaps the crew for a back link plus that list. Add a sub page by adding its folder and a dict entry +- `/admin/settings` (live) - redirects to `SettingsDefaultPath` (`/admin/settings/general`). Sub pages: `general/` (theme: System/Light/Dark per device via `useThemePreference`, stored in `localStorage[AppData.ThemeStorageKey]`, the same key `ThemeInitScript` and twui's `ColorSchemeSelector` read; applies instantly, no save bar), `profile/` (current user profile), `change-password/` (set/change own password) `models/` (super admin only, `super_admin_only` in the dict hides it from the rail: model providers and which model each harness uses) and `environment/` (super admin only: app settings and the envs bots and computers use, per scope), each wrapped in `SettingsShell` (`(partials)/settings-shell.tsx`: 64px pane header like the conversation header, then one centered column). Content is built from `SettingsGroup` blocks (label + description left, controls right, hairline-separated, no cards) and ends with the sticky `SettingsSaveBar` (Save enabled only when dirty, Discard, one-line status). Saves don't reload the page. Keep save notices in component state: `useFormInit` clears `status` whenever the form changes. `bots/` (the crew: chat, edit with `BotForm`, models, delete a worker via `api/admin/bots/delete-bot`), `workspaces/` (open, create through the Add flow, delete), `computers/` (state and resources, live; super admins create, edit with `AddComputerForm`, retry, delete), `mcps/` and `tools/` (see MCPs and custom tools: super admins add, edit, refresh or try, delete; admins only set Access, `extension-access-modal.tsx`; add, edit, access and try open in modals (`AddModalShell`)), those five for `canManageCrew` only (`grab-crew-settings-server-props.ts`, `can_view` in the dict) and `users/` (for `canListUsers`: super admins and admins; the dict's `can_view` gates it in the rail; add with a password (`canCreateUser`), edit (`canUpdateUser`), both with the one `UserForm` (`admin/people/create/(partials)/user-form/`, `useFormInit`, photo included, saved in one request), revoke (adds `revoked`, `canRevokeUser`), delete (`canRevokeUser`'s rules, never yourself), pick their user types from `UserTypes` (`user-types-select.tsx`: only the ones `filterAssignableUserTypes` allows, never `UneditableUserTypes`; at least one, since login needs a type; `saveUser` checks each grant and removal with `userTypeAssignment`; the super admin's types are never shown or changed there, and they can't be revoked; the password is entered twice, `SaveUserRequestSchema` checks they match), delete with everything theirs; `api/admin/users/save-user`, `delete-user`). The lists share `SettingsList`, `SettingsListRow`, `SettingsRowAction` and `use-settings-list`. Sub pages are declared once in `src/dict/settings-pages-dict.ts`; on every `/admin/settings/*` page the rail swaps the crew for a back link plus that list. Add a sub page by adding its folder and a dict entry Every `/admin/*` page renders in the bots layout (`src/layouts/bots/`); there is no separate admin layout or aside. The rail header's Add icon (super admins and admins only, like the rail's "+" buttons) opens the Add flow (`(partials)/add-flow.tsx`, `AddFlowContext`): a bot, a workspace or (super admins) a computer, each form in a modal stacked over the choice, so closing it goes back there (the Bots and Workspaces labels have a "+" that opens the form directly); on a workspace's pages new bots and computers join it. The rail shows the crew (`(partials)/rail-crew-list.tsx`, needs `bots` and `workspaces` in page props): the coordinator, the latest `AppData.RailRecentBots` workers (plus any open, unread or needing you, `pick-rail-bots`), "See all bots" with a few stones opening every bot in a modal, then Workspaces with a "+". On `/admin/workspaces/*` it shows the workspace's crew instead (`(partials)/rail-workspace-list.tsx`: back link, the workspace, then only the bots attached to it, "Workspace bots"; the coordinator stays in the main crew rail, and on a workspace bot's page the back link returns to the workspace page), except on `/admin/settings/*` where it shows "Back to bots" plus the settings sub pages (`(partials)/rail-settings-list.tsx`). Page heroes use the shared `AdminHero` component (`src/components/general/admin-hero.tsx`). @@ -187,7 +190,7 @@ Server functions use `index.server.ts` alongside `index.tsx` (e.g. `single-docum | Directory | Description | | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | | `@moduletrace/twui` | **The** design system - external package. Contains layout primitives, elements, editors (TinyMCE), form components, hooks, SVGs, and utilities. Import from `@moduletrace/twui` or a deep path such as `@moduletrace/twui/layout/Button` | -| `src/components/general/` | Shared page components built on twui: `logo`, `logo-icon`, `wordmark`, `bot-stone` (+ `bot-stone-face`, `stone-eye-*`), `user-avatar` (round photo or initials, `sm|md|lg`), `computer-view` (+ `computer-view-frame`, `computer-view-mode-button`), `inner-page-hero-section`, `main-hero-section`, `admin-hero`, `login-shell`, `tagline-section`, media/image helpers | +| `src/components/general/` | Shared page components built on twui: `logo`, `logo-icon`, `wordmark`, `bot-stone` (+ `bot-stone-face`, `stone-eye-*`), `user-avatar` (round photo or initials, `sm|md|lg`), `code-editor` (twui `AceEditor` for commands and code, loading the ACE scripts on demand via `loadAceScripts`), `icon-tile`, `computer-view` (+ `computer-view-frame`, `computer-view-mode-button`), `inner-page-hero-section`, `main-hero-section`, `admin-hero`, `login-shell`, `tagline-section`, media/image helpers | ## UI & Styling Rules (MANDATORY) diff --git a/db/schema.ts b/db/schema.ts index 61a6047..fe3bc00 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -20,6 +20,8 @@ import { BotJobKinds, BotJobUpdateKinds } from "@/src/dict/bot-queue-dict"; import { AppData } from "@/src/data/app-data"; import type { BUN_SQLITE_DatabaseSchemaType } from "@moduletrace/bun-sqlite/types"; import { EnvParadigms } from "@/src/dict/env-dict"; +import { McpServerStatuses, McpTransports } from "@/src/dict/mcp-dict"; +import { CustomToolKinds } from "@/src/dict/custom-tool-dict"; const schema: BUN_SQLITE_DatabaseSchemaType = { tables: [ @@ -1682,6 +1684,320 @@ const schema: BUN_SQLITE_DatabaseSchemaType = { }, ], }, + { + tableName: "mcp_servers", + tableDescription: `MCP servers bots can use (Settings, under MCPs). Per bot (bot_mcp_servers), per workspace (workspace_mcp_servers) or every bot (all_bots). Private: headers and env may hold literal secrets`, + fields: [ + { + fieldName: "name", + dataType: "TEXT", + }, + { + fieldName: "slug", + fieldDescription: `Prefixes its tools' names (__). Unique`, + dataType: "TEXT", + unique: true, + }, + { + fieldName: "description", + dataType: "TEXT", + }, + { + fieldName: "transport", + dataType: "TEXT", + options: McpTransports.map((t) => t.value), + defaultValue: "remote", + }, + { + fieldName: "url", + fieldDescription: `remote: the server's MCP endpoint`, + dataType: "TEXT", + }, + { + fieldName: "headers", + fieldDescription: `remote: request headers, {"Authorization":"Bearer {{env.KEY}}"}. Values are templates`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "command", + fieldDescription: `stdio: the command that starts the server, run with the host's shell in DATA_DIR/mcp/`, + dataType: "TEXT", + }, + { + fieldName: "setup_command", + fieldDescription: `stdio: installs what the command needs. Runs before the first start and again whenever it changes`, + dataType: "TEXT", + }, + { + fieldName: "env", + fieldDescription: `stdio: the only environment the server gets (besides PATH and HOME), {"GITHUB_TOKEN":"{{env.GITHUB_TOKEN}}"}. Values are templates`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "excluded_tools", + fieldDescription: `Tool names (the server's own) bots don't get`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "tools_cache", + fieldDescription: `The server's tools as last listed ([{name, description, input_schema}]). Runs build their tools from it and connect on the first call`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "all_bots", + fieldDescription: `1 gives it to every bot, the coordinator included`, + dataType: "INTEGER", + options: [0, 1], + defaultValue: 0, + }, + { + fieldName: "enabled", + dataType: "INTEGER", + options: [0, 1], + defaultValue: 1, + }, + { + fieldName: "status", + dataType: "TEXT", + options: McpServerStatuses.map((s) => s.value), + defaultValue: "untested", + }, + { + fieldName: "last_error", + dataType: "TEXT", + }, + { + fieldName: "last_synced_at", + fieldDescription: `Unix ms of the last tool listing`, + dataType: "INTEGER", + }, + { + fieldName: "user_id", + fieldDescription: `Who created it. Not a foreign key: it belongs to the install`, + dataType: "INTEGER", + }, + ], + }, + { + tableName: "custom_tools", + tableDescription: `Tools bots can call (Settings, under Tools): an HTTP request, a script on a computer, or a prompt. Per bot (bot_custom_tools), per workspace (workspace_custom_tools) or every bot (all_bots)`, + fields: [ + { + fieldName: "name", + fieldDescription: `What the model calls it: snake_case, unique, never a built-in tool's name`, + dataType: "TEXT", + unique: true, + }, + { + fieldName: "description", + fieldDescription: `What the model reads to decide when to call it`, + dataType: "TEXT", + }, + { + fieldName: "kind", + dataType: "TEXT", + options: CustomToolKinds.map((k) => k.value), + }, + { + fieldName: "parameters", + fieldDescription: `JSON schema (an object) of its arguments`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "config", + fieldDescription: `Per kind. http: {method, url, headers, body, timeout_ms}; script: {language, code, timeout_ms}; prompt: {prompt}. Text fields are templates ({{args.x}}, {{env.KEY}})`, + dataType: "TEXT", + json: true, + }, + { + fieldName: "all_bots", + fieldDescription: `1 gives it to every bot, the coordinator included`, + dataType: "INTEGER", + options: [0, 1], + defaultValue: 0, + }, + { + fieldName: "enabled", + dataType: "INTEGER", + options: [0, 1], + defaultValue: 1, + }, + { + fieldName: "user_id", + fieldDescription: `Who created it. Not a foreign key: it belongs to the install`, + dataType: "INTEGER", + }, + ], + }, + { + tableName: "bot_mcp_servers", + tableDescription: `A bot that gets an MCP server's tools everywhere it works`, + fields: [ + { + fieldName: "bot_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "bots", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_mcp_servers_bots_fk", + }, + }, + { + fieldName: "mcp_server_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "mcp_servers", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_mcp_servers_mcp_servers_fk", + }, + }, + ], + uniqueConstraints: [ + { + alias: "one_mcp_server_per_bot", + constraintName: "A bot gets an MCP server once", + constraintTableFields: [{ value: "bot_id" }, { value: "mcp_server_id" }], + }, + ], + indexes: [ + { + indexName: "idx_bot_mcp_servers_mcp_server_id", + indexTableFields: ["mcp_server_id"], + }, + ], + }, + { + tableName: "workspace_mcp_servers", + tableDescription: `Every bot in the workspace gets the MCP server's tools there`, + fields: [ + { + fieldName: "workspace_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "workspaces", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "workspace_mcp_servers_workspaces_fk", + }, + }, + { + fieldName: "mcp_server_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "mcp_servers", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "workspace_mcp_servers_mcp_servers_fk", + }, + }, + ], + uniqueConstraints: [ + { + alias: "one_mcp_server_per_workspace", + constraintName: "A workspace gets an MCP server once", + constraintTableFields: [{ value: "workspace_id" }, { value: "mcp_server_id" }], + }, + ], + indexes: [ + { + indexName: "idx_workspace_mcp_servers_mcp_server_id", + indexTableFields: ["mcp_server_id"], + }, + ], + }, + { + tableName: "bot_custom_tools", + tableDescription: `A bot that gets a custom tool everywhere it works`, + fields: [ + { + fieldName: "bot_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "bots", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_custom_tools_bots_fk", + }, + }, + { + fieldName: "custom_tool_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "custom_tools", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "bot_custom_tools_custom_tools_fk", + }, + }, + ], + uniqueConstraints: [ + { + alias: "one_custom_tool_per_bot", + constraintName: "A bot gets a custom tool once", + constraintTableFields: [{ value: "bot_id" }, { value: "custom_tool_id" }], + }, + ], + indexes: [ + { + indexName: "idx_bot_custom_tools_custom_tool_id", + indexTableFields: ["custom_tool_id"], + }, + ], + }, + { + tableName: "workspace_custom_tools", + tableDescription: `Every bot in the workspace gets the custom tool there`, + fields: [ + { + fieldName: "workspace_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "workspaces", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "workspace_custom_tools_workspaces_fk", + }, + }, + { + fieldName: "custom_tool_id", + dataType: "INTEGER", + foreignKey: { + cascadeDelete: true, + destinationTableName: "custom_tools", + destinationTableColumnName: "id", + destinationTableColumnType: "INTEGER", + foreignKeyName: "workspace_custom_tools_custom_tools_fk", + }, + }, + ], + uniqueConstraints: [ + { + alias: "one_custom_tool_per_workspace", + constraintName: "A workspace gets a custom tool once", + constraintTableFields: [{ value: "workspace_id" }, { value: "custom_tool_id" }], + }, + ], + indexes: [ + { + indexName: "idx_workspace_custom_tools_custom_tool_id", + indexTableFields: ["custom_tool_id"], + }, + ], + }, { tableName: "logs", fields: [ diff --git a/db/types/db.ts b/db/types/db.ts index 6b6f984..2ef119d 100644 --- a/db/types/db.ts +++ b/db/types/db.ts @@ -29,6 +29,12 @@ export const BunSQLiteTables = [ "harness_settings", "bot_harness_settings", "workspace_harness_settings", + "mcp_servers", + "custom_tools", + "bot_mcp_servers", + "workspace_mcp_servers", + "bot_custom_tools", + "workspace_custom_tools", "logs", "envs", ] as const @@ -1026,6 +1032,180 @@ export type BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS = { options?: string; } +export type BUN_SQLITE_COBBLE_MCP_SERVERS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + name?: string; + /** + * Prefixes its tools' names (__). Unique + */ + slug?: string; + description?: string; + transport?: "remote" | "stdio" | ""; + /** + * remote: the server's MCP endpoint + */ + url?: string; + /** + * remote: request headers, {"Authorization":"Bearer {{env.KEY}}"}. Values are templates + */ + headers?: string; + /** + * stdio: the command that starts the server, run with the host's shell in DATA_DIR/mcp/ + */ + command?: string; + /** + * stdio: installs what the command needs. Runs before the first start and again whenever it changes + */ + setup_command?: string; + /** + * stdio: the only environment the server gets (besides PATH and HOME), {"GITHUB_TOKEN":"{{env.GITHUB_TOKEN}}"}. Values are templates + */ + env?: string; + /** + * Tool names (the server's own) bots don't get + */ + excluded_tools?: string; + /** + * The server's tools as last listed ([{name, description, input_schema}]). Runs build their tools from it and connect on the first call + */ + tools_cache?: string; + /** + * 1 gives it to every bot, the coordinator included + */ + all_bots?: 0 | 1 | ""; + enabled?: 0 | 1 | ""; + status?: "untested" | "ok" | "error" | ""; + last_error?: string; + /** + * Unix ms of the last tool listing + */ + last_synced_at?: number | ""; + /** + * Who created it. Not a foreign key: it belongs to the install + */ + user_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_CUSTOM_TOOLS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + /** + * What the model calls it: snake_case, unique, never a built-in tool's name + */ + name?: string; + /** + * What the model reads to decide when to call it + */ + description?: string; + kind?: "http" | "script" | "prompt" | ""; + /** + * JSON schema (an object) of its arguments + */ + parameters?: string; + /** + * Per kind. http: {method, url, headers, body, timeout_ms}; script: {language, code, timeout_ms}; prompt: {prompt}. Text fields are templates ({{args.x}}, {{env.KEY}}) + */ + config?: string; + /** + * 1 gives it to every bot, the coordinator included + */ + all_bots?: 0 | 1 | ""; + enabled?: 0 | 1 | ""; + /** + * Who created it. Not a foreign key: it belongs to the install + */ + user_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_BOT_MCP_SERVERS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + bot_id?: number | ""; + mcp_server_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + workspace_id?: number | ""; + mcp_server_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + bot_id?: number | ""; + custom_tool_id?: number | ""; +} + +export type BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS = { + /** + * The unique identifier of the record. + */ + id?: number | ""; + /** + * The time when the record was created. (Unix Timestamp) + */ + created_at?: number | ""; + /** + * The time when the record was updated. (Unix Timestamp) + */ + updated_at?: number | ""; + workspace_id?: number | ""; + custom_tool_id?: number | ""; +} + export type BUN_SQLITE_COBBLE_LOGS = { /** * The unique identifier of the record. @@ -1120,4 +1300,4 @@ export type BUN_SQLITE_COBBLE_ENVS = { expose_to_computers?: 0 | 1 | ""; } -export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_ENVS \ No newline at end of file +export type BUN_SQLITE_COBBLE_ALL_TYPEDEFS = BUN_SQLITE_COBBLE_USERS & BUN_SQLITE_COBBLE_USER_TYPES & BUN_SQLITE_COBBLE_SSO_LOGIN_CODES & BUN_SQLITE_COBBLE_USER_PASSWORDS & BUN_SQLITE_COBBLE_MEDIA & BUN_SQLITE_COBBLE_MEDIA_PARADIGMS & BUN_SQLITE_COBBLE_BOTS & BUN_SQLITE_COBBLE_BOT_SESSIONS & BUN_SQLITE_COBBLE_BOT_MESSAGES & BUN_SQLITE_COBBLE_BOT_TASKS & BUN_SQLITE_COBBLE_BOT_TASK_STEPS & BUN_SQLITE_COBBLE_BOTS_QUEUE & BUN_SQLITE_COBBLE_BOTS_QUEUE_UPDATES & BUN_SQLITE_COBBLE_CRON_JOBS & BUN_SQLITE_COBBLE_WORKSPACES & BUN_SQLITE_COBBLE_WORKSPACE_BOTS & BUN_SQLITE_COBBLE_WORKSPACE_USERS & BUN_SQLITE_COBBLE_BOT_USERS & BUN_SQLITE_COBBLE_BOT_USER_MEMORIES & BUN_SQLITE_COBBLE_BOT_USER_FACES & BUN_SQLITE_COBBLE_BOT_MESSAGE_MEDIA & BUN_SQLITE_COBBLE_CONTAINERS & BUN_SQLITE_COBBLE_COMPUTERS & BUN_SQLITE_COBBLE_BOT_COMPUTERS & BUN_SQLITE_COBBLE_WORKSPACE_COMPUTERS & BUN_SQLITE_COBBLE_HARNESS_PROVIDERS & BUN_SQLITE_COBBLE_HARNESS_MODELS & BUN_SQLITE_COBBLE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_BOT_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_WORKSPACE_HARNESS_SETTINGS & BUN_SQLITE_COBBLE_MCP_SERVERS & BUN_SQLITE_COBBLE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_BOT_MCP_SERVERS & BUN_SQLITE_COBBLE_WORKSPACE_MCP_SERVERS & BUN_SQLITE_COBBLE_BOT_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_WORKSPACE_CUSTOM_TOOLS & BUN_SQLITE_COBBLE_LOGS & BUN_SQLITE_COBBLE_ENVS \ No newline at end of file diff --git a/src/bots-queue/run-bot-reply.ts b/src/bots-queue/run-bot-reply.ts index fec9a57..5a8fd53 100644 --- a/src/bots-queue/run-bot-reply.ts +++ b/src/bots-queue/run-bot-reply.ts @@ -11,6 +11,7 @@ import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers"; import runTextLoop from "@/src/harness/text/run-text-loop"; import grabBotChatTools from "@/src/harness/tools/bot/grab-bot-chat-tools"; +import openBotExtensions from "@/src/harness/tools/extensions/open-bot-extensions"; import botMessagesToHarness from "./bot-messages-to-harness"; import composeComputersNote from "./compose-computers-note"; import grabSessionCarryOver from "./grab-session-carry-over"; @@ -33,7 +34,9 @@ type Params = BotJobContext & { * knowing where it is (its workspace, if the session is in one), what * happened in the session before (`grabSessionCarryOver`) and its * computers (`composeComputersNote`), with its chat tools (memory, - * instructions, start_task, and for the coordinator delegate_task). + * instructions, start_task, and for the coordinator delegate_task) plus + * its MCP servers' and custom tools (`openBotExtensions`, script tools on + * its current computer). * Finishes silently when the last message is already the bot's, i.e. an * earlier reply answered everything. */ @@ -82,6 +85,14 @@ export default async function runBotReply({ onTakeOver: ({ ref }) => take_overs.push(ref), }); + const extensions = await openBotExtensions({ + bot_id: bot.id, + workspace_id, + computer_id: computers[0]?.id || null, + taken: tools.map((tool) => tool.name), + signal, + }); + /** * delegate_task is only there while the coordinator has workers. */ @@ -92,33 +103,38 @@ export default async function runBotReply({ ? CoordinatorReplyNote : CoordinatorSoloNote; - const result = await runTextLoop({ - system: [ - prompt, - carry_over, - ChatReplyNote, - workspace_id ? WorkspaceLearningNote : "", - composeComputersNote({ - computers, - is_coordinator: bot.kind == "coordinator", - in_workspace: Boolean(workspace_id), + try { + const result = await runTextLoop({ + system: [ + prompt, + carry_over, + ChatReplyNote, + workspace_id ? WorkspaceLearningNote : "", + composeComputersNote({ + computers, + is_coordinator: bot.kind == "coordinator", + in_workspace: Boolean(workspace_id), + }), + crew_note, + extensions.note, + `Now: ${new Date().toUTCString()}.`, + ] + .filter(Boolean) + .join("\n\n"), + messages: botMessagesToHarness({ + messages: history, + parts: await loadAttachmentParts({ messages: history }), }), - crew_note, - `Now: ${new Date().toUTCString()}.`, - ] - .filter(Boolean) - .join("\n\n"), - messages: botMessagesToHarness({ - messages: history, - parts: await loadAttachmentParts({ messages: history }), - }), - tools, - max_steps: AppData.BotReplyMaxToolSteps, - bot_id: bot.id, - workspace_id, - signal, - onStep: report, - }); + tools: [...tools, ...extensions.tools], + max_steps: AppData.BotReplyMaxToolSteps, + bot_id: bot.id, + workspace_id, + signal, + onStep: report, + }); - return withTakeOverRefs({ text: result.text, refs: take_overs }); + return withTakeOverRefs({ text: result.text, refs: take_overs }); + } finally { + await extensions.close(); + } } diff --git a/src/bots-queue/run-bot-task.ts b/src/bots-queue/run-bot-task.ts index d7d22f5..22f516a 100644 --- a/src/bots-queue/run-bot-task.ts +++ b/src/bots-queue/run-bot-task.ts @@ -5,6 +5,7 @@ import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import grabUserWorkspaces from "@/src/functions/backend/db/workspaces/grab-user-workspaces"; import runTextLoop from "@/src/harness/text/run-text-loop"; import openComputerTools from "@/src/harness/tools/computer/open-computer-tools"; +import openBotExtensions, { type OpenBotExtensions } from "@/src/harness/tools/extensions/open-bot-extensions"; import grabBotTools from "@/src/harness/tools/bot/grab-bot-tools"; import createWorkspaceTool from "@/src/harness/tools/bot/create-workspace-tool"; import updateWorkspaceTool from "@/src/harness/tools/bot/update-workspace-tool"; @@ -28,8 +29,9 @@ type Params = BotJobContext & { /** * Works through a task in its session's workspace (if any): the bot * follows its brief with its full prompt, - * its own tools, search_computers across all its computers and, when the - * job has a computer, that computer's tools (over MCP), until it answers. That answer is the report for the user. + * its own tools, search_computers across all its computers, its MCP + * servers' and custom tools (`openBotExtensions`) and, when the job has a + * computer, that computer's tools (over MCP), until it answers. That answer is the report for the user. * When only the user can go on (a login, a captcha), the bot asks them to * take over (`ask_user_to_take_over`) and the report needs them. * Every model turn is reported as a step (`report`). @@ -53,6 +55,8 @@ export default async function runBotTask({ }) : null; + let extensions: OpenBotExtensions | undefined; + try { const session_id = await jobSessionId({ job }); const prompt = await grabBotPrompt({ bot_id: bot.id, user_id, workspace_id }); @@ -82,6 +86,43 @@ export default async function runBotTask({ ...computers.filter((c) => c.id != job.computer_id), ]; + const tools = [ + ...grabBotTools({ user_id, bot_id: bot.id, workspace_id, computers }), + ...(bot.kind == "coordinator" + ? [ + createWorkspaceTool({ user_id, workspaces: coordinator_workspaces }), + ...(coordinator_workspaces.length + ? [ + updateWorkspaceTool({ + user_id, + workspaces: coordinator_workspaces, + workspace_id, + }), + ] + : []), + ] + : []), + ...(computers.length + ? [ + searchComputersTool({ computers, workspace_id }), + askToTakeOverTool({ + computers: take_over_computers, + in_task: true, + onAsk: ({ ref }) => take_overs.push(ref), + }), + ] + : []), + ...(computer?.tools || []), + ]; + + extensions = await openBotExtensions({ + bot_id: bot.id, + workspace_id, + computer_id: job.computer_id || null, + taken: tools.map((tool) => tool.name), + signal, + }); + const result = await runTextLoop({ system: [ prompt, @@ -104,6 +145,7 @@ export default async function runBotTask({ .map((env) => `- ${env.key}${env.description ? `: ${env.description}` : ""}`) .join("\n")}` : "", + extensions.note, ] .filter(Boolean) .join("\n\n"), @@ -117,34 +159,7 @@ export default async function runBotTask({ }), }, ], - tools: [ - ...grabBotTools({ user_id, bot_id: bot.id, workspace_id, computers }), - ...(bot.kind == "coordinator" - ? [ - createWorkspaceTool({ user_id, workspaces: coordinator_workspaces }), - ...(coordinator_workspaces.length - ? [ - updateWorkspaceTool({ - user_id, - workspaces: coordinator_workspaces, - workspace_id, - }), - ] - : []), - ] - : []), - ...(computers.length - ? [ - searchComputersTool({ computers, workspace_id }), - askToTakeOverTool({ - computers: take_over_computers, - in_task: true, - onAsk: ({ ref }) => take_overs.push(ref), - }), - ] - : []), - ...(computer?.tools || []), - ], + tools: [...tools, ...extensions.tools], bot_id: bot.id, workspace_id, signal, @@ -157,6 +172,7 @@ export default async function runBotTask({ refs: take_overs, }); } finally { + await extensions?.close(); await computer?.close(); } } diff --git a/src/components/general/code-editor.tsx b/src/components/general/code-editor.tsx new file mode 100644 index 0000000..5442fa4 --- /dev/null +++ b/src/components/general/code-editor.tsx @@ -0,0 +1,61 @@ +import { useEffect, useRef } from "react"; +import AceEditor from "@moduletrace/twui/editors/AceEditor"; +import type AceEditorModes from "@moduletrace/twui/editors/ace-editor-modes"; +import Span from "@moduletrace/twui/layout/Span"; +import Stack from "@moduletrace/twui/layout/Stack"; +import { twMerge } from "tailwind-merge"; +import loadAceScripts from "@/src/utils/load-ace-scripts"; + +type Props = { + title: string; + /** + * The starting text. Changes come back through `onChange`. + */ + content?: string; + mode: (typeof AceEditorModes)[number]; + placeholder?: string; + /** + * One line under it, like a field's info. + */ + info?: string; + /** + * Its height, e.g. `h-24`. + */ + className?: string; + onChange: (value: string) => void; +}; + +/** + * A code field: twui's `AceEditor` with its label, and a line of info + * under it. Loads the editor's scripts when it first shows + * (`loadAceScripts`). + */ +export default function CodeEditor({ title, content, mode, placeholder, info, className, onChange }: Props) { + /** + * The editor keeps the handler it was created with; this keeps it + * calling the latest one. + */ + const change = useRef(onChange); + change.current = onChange; + + useEffect(() => { + loadAceScripts().catch(() => {}); + }, []); + + return ( + + change.current(value)} + /> + {info ? {info} : null} + + ); +} diff --git a/src/components/general/computer-icon-tile.tsx b/src/components/general/computer-icon-tile.tsx index acd5a49..c41ce17 100644 --- a/src/components/general/computer-icon-tile.tsx +++ b/src/components/general/computer-icon-tile.tsx @@ -1,13 +1,9 @@ -import Span from "@moduletrace/twui/layout/Span"; import { Monitor } from "lucide-react"; +import IconTile from "./icon-tile"; /** * A computer's avatar in lists: the monitor icon on a small tile. */ export default function ComputerIconTile() { - return ( - - - - ); + return ; } diff --git a/src/components/general/icon-tile.tsx b/src/components/general/icon-tile.tsx new file mode 100644 index 0000000..9565de0 --- /dev/null +++ b/src/components/general/icon-tile.tsx @@ -0,0 +1,17 @@ +import Span from "@moduletrace/twui/layout/Span"; +import type { LucideIcon } from "lucide-react"; + +type Props = { + Icon: LucideIcon; +}; + +/** + * An avatar in lists for what has no picture: its icon on a small tile. + */ +export default function IconTile({ Icon }: Props) { + return ( + + + + ); +} diff --git a/src/cron/index.ts b/src/cron/index.ts index e14e459..cfa9467 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -3,6 +3,7 @@ import cronHeartbeat from "./heartbeat"; import pruneBotsQueue from "./prune-bots-queue"; import cronSyncHarnessModels from "./sync-harness-models"; import syncComputers from "./sync-computers"; +import syncMcpServers from "./sync-mcp-servers"; import runCronJobs from "./run-cron-jobs"; import registerCronJob from "./register-cron-job"; @@ -40,6 +41,12 @@ export default function cron() { run_on_start: true, }); + registerCronJob({ + title: "sync-mcp-servers", + schedule: "@hourly", + task: syncMcpServers, + }); + registerCronJob({ title: "run-cron-jobs", schedule: "* * * * *", diff --git a/src/cron/sync-mcp-servers/index.ts b/src/cron/sync-mcp-servers/index.ts new file mode 100644 index 0000000..1804081 --- /dev/null +++ b/src/cron/sync-mcp-servers/index.ts @@ -0,0 +1,21 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import syncMcpServer from "@/src/functions/backend/db/extensions/sync-mcp-server"; + +/** + * Lists every enabled remote MCP server's tools again (`syncMcpServer`), + * so bots get tools the server added and lose ones it dropped. stdio + * servers are left alone: listing them means starting them, and they're + * refreshed when saved or from Settings, under MCPs. + */ +export default async function syncMcpServers() { + const rows = + ( + await BunSQLite.sql<{ id: number }>({ + sql: `SELECT id FROM mcp_servers WHERE enabled = 1 AND transport = 'remote'`, + }) + ).payload || []; + + for (const row of rows) { + await syncMcpServer({ mcp_server_id: row.id }); + } +} diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 7af7918..e2684a8 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -181,6 +181,22 @@ export const AppData = { * One MCP tool call gives up after this long without progress. */ McpToolTimeoutMs: 5 * 60 * 1000, + /** + * MCP servers set up in Settings, under MCPs: a stdio server runs in + * `DATA_DIR//` (its setup installs there, so it + * survives updates), its setup command may take `McpSetupTimeoutMs`, + * and it's stopped after `McpStdioIdleMs` without a call. + */ + McpDataDir: "mcp", + McpSetupTimeoutMs: 10 * 60 * 1000, + McpStdioIdleMs: 10 * 60 * 1000, + /** + * A custom tool (Settings, under Tools) gives up after its own timeout, + * `CustomToolDefaultTimeoutMs` when it has none, never past + * `CustomToolMaxTimeoutMs`. + */ + CustomToolDefaultTimeoutMs: 60 * 1000, + CustomToolMaxTimeoutMs: 10 * 60 * 1000, /** * `search_computers`: how long the command may run on each computer, * and how much of each computer's output the bot reads. diff --git a/src/dict/custom-tool-dict.ts b/src/dict/custom-tool-dict.ts new file mode 100644 index 0000000..9b244b9 --- /dev/null +++ b/src/dict/custom-tool-dict.ts @@ -0,0 +1,66 @@ +/** + * What a custom tool does when a bot calls it. `http` calls an API from the + * app, `script` runs code on the run's computer, `prompt` runs one model + * turn with a filled-in prompt. + */ +export const CustomToolKinds = [ + { + title: "HTTP request", + value: "http", + description: "Calls an API from the app and returns the response.", + }, + { + title: "Script on a computer", + value: "script", + description: "Runs code on the bot's computer and returns what it prints.", + }, + { + title: "Prompt", + value: "prompt", + description: "Runs one model turn with a filled-in prompt and returns the answer.", + }, +] as const; + +export type CustomToolKind = (typeof CustomToolKinds)[number]["value"]; + +export const CustomToolHttpMethods = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const; + +export type CustomToolHttpMethod = (typeof CustomToolHttpMethods)[number]; + +/** + * Languages a script tool runs in, with the interpreter on the computer + * and the code editor's mode. + */ +export const CustomToolScriptLanguages = [ + { title: "Bash", value: "bash", interpreter: "bash", extension: "sh", editor_mode: "sh" }, + { title: "Python", value: "python", interpreter: "python3", extension: "py", editor_mode: "python" }, + { title: "JavaScript", value: "javascript", interpreter: "node", extension: "js", editor_mode: "javascript" }, +] as const; + +export type CustomToolScriptLanguage = (typeof CustomToolScriptLanguages)[number]["value"]; + +/** + * Names the built-in bot tools use. A custom tool can't take one, and + * names starting with a computer's tool prefixes are kept for them too. + */ +export const ReservedToolNames = [ + "add_bot_to_workspace", + "ask_user_to_take_over", + "assign_computer", + "create_bot", + "create_schedule", + "create_workspace", + "delegate_task", + "delete_schedule", + "list_schedules", + "search_computers", + "set_bot_workspace_instructions", + "set_workspace_computer", + "start_task", + "update_instructions", + "update_memory", + "update_schedule", + "update_workspace", +] as const; + +export const ReservedToolPrefixes = ["sandbox_", "browser_"] as const; diff --git a/src/dict/mcp-dict.ts b/src/dict/mcp-dict.ts new file mode 100644 index 0000000..c6bf53e --- /dev/null +++ b/src/dict/mcp-dict.ts @@ -0,0 +1,39 @@ +/** + * How the app reaches an MCP server. `remote`: over HTTP (Streamable HTTP, + * SSE fallback). `stdio`: a command the app starts on its own machine (the + * host in dev, the app's container in production) and talks to over its + * stdin and stdout. + */ +export const McpTransports = [ + { + title: "Remote (URL)", + value: "remote", + }, + { + title: "Command on this server (stdio)", + value: "stdio", + }, +] as const; + +export type McpTransport = (typeof McpTransports)[number]["value"]; + +/** + * The last time the app listed a server's tools: `ok`, `error` (see its + * `last_error`), or `untested` before the first time. + */ +export const McpServerStatuses = [ + { + title: "Untested", + value: "untested", + }, + { + title: "Connected", + value: "ok", + }, + { + title: "Error", + value: "error", + }, +] as const; + +export type McpServerStatus = (typeof McpServerStatuses)[number]["value"]; diff --git a/src/dict/private-tables.ts b/src/dict/private-tables.ts index e3ed5dd..780babc 100644 --- a/src/dict/private-tables.ts +++ b/src/dict/private-tables.ts @@ -2,7 +2,7 @@ import type { TableType } from "@/src/types"; /** * Tables holding secrets (password hashes, pending login codes, model - * provider keys, envs). They are never reachable through the admin CRUD API, not + * provider keys, envs, MCP servers' headers and env). They are never reachable through the admin CRUD API, not * even by super admins. */ export const PrivateTables: TableType[] = [ @@ -10,4 +10,5 @@ export const PrivateTables: TableType[] = [ "sso_login_codes", "harness_providers", "envs", + "mcp_servers", ]; diff --git a/src/dict/settings-pages-dict.ts b/src/dict/settings-pages-dict.ts index 9e87624..6f3b097 100644 --- a/src/dict/settings-pages-dict.ts +++ b/src/dict/settings-pages-dict.ts @@ -4,10 +4,12 @@ import { KeyRound, Layers, Monitor, + Plug, Users, SlidersHorizontal, UserRound, Variable, + Wrench, type LucideIcon, } from "lucide-react"; import type { BUN_SQLITE_COBBLE_USER_TYPES } from "@/db/types/db"; @@ -49,6 +51,8 @@ export const SettingsPages: SettingsPage[] = [ { title: "Bots", url: "/admin/settings/bots", Icon: Bot, can_view: canManageCrew }, { title: "Workspaces", url: "/admin/settings/workspaces", Icon: Layers, can_view: canManageCrew }, { title: "Computers", url: SettingsComputersPath, Icon: Monitor, can_view: canManageCrew }, + { title: "MCPs", url: "/admin/settings/mcps", Icon: Plug, can_view: canManageCrew }, + { title: "Tools", url: "/admin/settings/tools", Icon: Wrench, can_view: canManageCrew }, { title: "Users", url: "/admin/settings/users", diff --git a/src/functions/backend/db/extensions/delete-custom-tool.ts b/src/functions/backend/db/extensions/delete-custom-tool.ts new file mode 100644 index 0000000..c4ffe38 --- /dev/null +++ b/src/functions/backend/db/extensions/delete-custom-tool.ts @@ -0,0 +1,18 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import grabCustomTool from "./grab-custom-tool"; + +type Params = { + custom_tool_id: number | string; +}; + +/** + * Deletes a custom tool; its attachments go with it. + */ +export default async function deleteCustomTool({ custom_tool_id }: Params) { + const tool = await grabCustomTool({ custom_tool_id }); + const res = await BunSQLite.sql({ sql: `DELETE FROM custom_tools WHERE id = ?`, values: [tool.id] }); + + if (!res.success) throw new Error(`Couldn't delete the tool: ${res.error}`); + + return tool; +} diff --git a/src/functions/backend/db/extensions/delete-mcp-server.ts b/src/functions/backend/db/extensions/delete-mcp-server.ts new file mode 100644 index 0000000..e896366 --- /dev/null +++ b/src/functions/backend/db/extensions/delete-mcp-server.ts @@ -0,0 +1,23 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { stopStdioMcpServer } from "@/src/harness/mcp/mcp-stdio-pool"; +import grabMcpServer from "./grab-mcp-server"; + +type Params = { + mcp_server_id: number | string; +}; + +/** + * Deletes an MCP server (its attachments go with it) and stops it if it's + * running here. Its directory, with whatever its setup installed, stays. + */ +export default async function deleteMcpServer({ mcp_server_id }: Params) { + const server = await grabMcpServer({ mcp_server_id }); + + stopStdioMcpServer({ server_id: server.id }); + + const res = await BunSQLite.sql({ sql: `DELETE FROM mcp_servers WHERE id = ?`, values: [server.id] }); + + if (!res.success) throw new Error(`Couldn't delete the MCP server: ${res.error}`); + + return server; +} diff --git a/src/functions/backend/db/extensions/extension-tables.ts b/src/functions/backend/db/extensions/extension-tables.ts new file mode 100644 index 0000000..c5da60b --- /dev/null +++ b/src/functions/backend/db/extensions/extension-tables.ts @@ -0,0 +1,22 @@ +/** + * Where each kind of extension lives: its table, and the link tables + * that attach it to bots and workspaces. + */ +export const ExtensionTables = { + mcp: { + table: "mcp_servers", + bot_link: "bot_mcp_servers", + workspace_link: "workspace_mcp_servers", + column: "mcp_server_id", + label: "MCP server", + }, + tool: { + table: "custom_tools", + bot_link: "bot_custom_tools", + workspace_link: "workspace_custom_tools", + column: "custom_tool_id", + label: "tool", + }, +} as const; + +export type ExtensionKind = keyof typeof ExtensionTables; diff --git a/src/functions/backend/db/extensions/grab-bot-extensions.ts b/src/functions/backend/db/extensions/grab-bot-extensions.ts new file mode 100644 index 0000000..f7c9596 --- /dev/null +++ b/src/functions/backend/db/extensions/grab-bot-extensions.ts @@ -0,0 +1,26 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_CUSTOM_TOOLS, BUN_SQLITE_COBBLE_MCP_SERVERS } from "@/db/types/db"; +import botExtensionsSql from "@/src/utils/bot-extensions-sql"; +import parseCustomToolRow from "./parse-custom-tool-row"; +import parseMcpServerRow from "./parse-mcp-server-row"; + +type Params = { + bot_id: number | string; + workspace_id?: number | string | null; +}; + +/** + * The MCP servers and custom tools a bot gets on one run + * (`botExtensionsSql`): every bot's, its own, and its workspace's there. + */ +export default async function grabBotExtensions({ bot_id, workspace_id }: Params) { + const [servers, tools] = await Promise.all([ + BunSQLite.sql(botExtensionsSql({ kind: "mcp", bot_id, workspace_id })), + BunSQLite.sql(botExtensionsSql({ kind: "tool", bot_id, workspace_id })), + ]); + + return { + servers: (servers.payload || []).map((row) => parseMcpServerRow({ row })), + tools: (tools.payload || []).map((row) => parseCustomToolRow({ row })), + }; +} diff --git a/src/functions/backend/db/extensions/grab-custom-tool.ts b/src/functions/backend/db/extensions/grab-custom-tool.ts new file mode 100644 index 0000000..54e29cb --- /dev/null +++ b/src/functions/backend/db/extensions/grab-custom-tool.ts @@ -0,0 +1,24 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_CUSTOM_TOOLS } from "@/db/types/db"; +import AppError from "@/src/utils/app-error"; +import parseCustomToolRow from "./parse-custom-tool-row"; + +type Params = { + custom_tool_id: number | string; +}; + +/** + * One custom tool, or an `AppError` when there's none. + */ +export default async function grabCustomTool({ custom_tool_id }: Params) { + const row = ( + await BunSQLite.sql({ + sql: `SELECT * FROM custom_tools WHERE id = ?`, + values: [custom_tool_id], + }) + ).payload?.[0]; + + if (!row) throw new AppError(`That tool doesn't exist.`); + + return parseCustomToolRow({ row }); +} diff --git a/src/functions/backend/db/extensions/grab-extension-access.ts b/src/functions/backend/db/extensions/grab-extension-access.ts new file mode 100644 index 0000000..2ccd446 --- /dev/null +++ b/src/functions/backend/db/extensions/grab-extension-access.ts @@ -0,0 +1,33 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { ExtensionTables, type ExtensionKind } from "./extension-tables"; + +type Params = { + kind: ExtensionKind; +}; + +/** + * The bots and workspaces every MCP server (or tool) is attached to, by + * its id. + */ +export default async function grabExtensionAccess({ kind }: Params) { + const { bot_link, workspace_link, column } = ExtensionTables[kind]; + const [bots, workspaces] = await Promise.all([ + BunSQLite.sql<{ id: number; target: number }>({ + sql: `SELECT ${column} AS id, bot_id AS target FROM ${bot_link}`, + }), + BunSQLite.sql<{ id: number; target: number }>({ + sql: `SELECT ${column} AS id, workspace_id AS target FROM ${workspace_link}`, + }), + ]); + + const access = new Map(); + const entry = (id: number) => { + if (!access.has(id)) access.set(id, { bot_ids: [], workspace_ids: [] }); + return access.get(id)!; + }; + + for (const row of bots.payload || []) entry(Number(row.id)).bot_ids.push(Number(row.target)); + for (const row of workspaces.payload || []) entry(Number(row.id)).workspace_ids.push(Number(row.target)); + + return (id: number) => access.get(id) || { bot_ids: [], workspace_ids: [] }; +} diff --git a/src/functions/backend/db/extensions/grab-extension-targets.ts b/src/functions/backend/db/extensions/grab-extension-targets.ts new file mode 100644 index 0000000..966db4a --- /dev/null +++ b/src/functions/backend/db/extensions/grab-extension-targets.ts @@ -0,0 +1,39 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_BOTS, BUN_SQLITE_COBBLE_WORKSPACES } from "@/db/types/db"; +import type { ExtensionTargets } from "@/src/types"; +import type { CrewRole } from "@/src/utils/rbac/crew-role"; + +type Params = { + role: CrewRole; +}; + +/** + * What an MCP server or tool can be attached to: every worker (and the + * coordinator, for the super admin, who alone reaches it) and every + * workspace not archived. Only the super admin edits. + */ +export default async function grabExtensionTargets({ role }: Params): Promise { + const [bots, workspaces] = await Promise.all([ + BunSQLite.sql({ + sql: `SELECT id, name, role, kind FROM bots ${role == "super_admin" ? "" : `WHERE kind = 'worker'`} + ORDER BY kind = 'coordinator' DESC, name COLLATE NOCASE ASC, id ASC`, + }), + BunSQLite.sql({ + sql: `SELECT id, name FROM workspaces WHERE archived = 0 ORDER BY name COLLATE NOCASE ASC, id ASC`, + }), + ]); + + return { + bots: (bots.payload || []).map((bot) => ({ + id: Number(bot.id), + name: bot.name || `Bot ${bot.id}`, + role: bot.role || "", + kind: bot.kind || "worker", + })), + workspaces: (workspaces.payload || []).map((workspace) => ({ + id: Number(workspace.id), + name: workspace.name || `Workspace ${workspace.id}`, + })), + can_edit: role == "super_admin", + }; +} diff --git a/src/functions/backend/db/extensions/grab-mcp-server.ts b/src/functions/backend/db/extensions/grab-mcp-server.ts new file mode 100644 index 0000000..b992ca4 --- /dev/null +++ b/src/functions/backend/db/extensions/grab-mcp-server.ts @@ -0,0 +1,24 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_MCP_SERVERS } from "@/db/types/db"; +import AppError from "@/src/utils/app-error"; +import parseMcpServerRow from "./parse-mcp-server-row"; + +type Params = { + mcp_server_id: number | string; +}; + +/** + * One MCP server, or an `AppError` when there's none. + */ +export default async function grabMcpServer({ mcp_server_id }: Params) { + const row = ( + await BunSQLite.sql({ + sql: `SELECT * FROM mcp_servers WHERE id = ?`, + values: [mcp_server_id], + }) + ).payload?.[0]; + + if (!row) throw new AppError(`That MCP server doesn't exist.`); + + return parseMcpServerRow({ row }); +} diff --git a/src/functions/backend/db/extensions/grab-mcps-overview.ts b/src/functions/backend/db/extensions/grab-mcps-overview.ts new file mode 100644 index 0000000..73bf7b2 --- /dev/null +++ b/src/functions/backend/db/extensions/grab-mcps-overview.ts @@ -0,0 +1,38 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_MCP_SERVERS } from "@/db/types/db"; +import type { McpServersOverviewPayload } from "@/src/types"; +import grabCrewRole from "../access/grab-crew-role"; +import grabExtensionAccess from "./grab-extension-access"; +import grabExtensionTargets from "./grab-extension-targets"; +import parseMcpServerRow from "./parse-mcp-server-row"; + +type Params = { + user_id: number | string; +}; + +/** + * The MCPs settings page: every MCP server with who gets it, and what it + * can be attached to. How a server is reached (URL, headers, command, + * env) only goes to the super admin. + */ +export default async function grabMcpsOverview({ user_id }: Params): Promise { + const role = await grabCrewRole({ user_id }); + const [rows, access, targets] = await Promise.all([ + BunSQLite.sql({ sql: `SELECT * FROM mcp_servers ORDER BY name COLLATE NOCASE ASC, id ASC` }), + grabExtensionAccess({ kind: "mcp" }), + grabExtensionTargets({ role }), + ]); + + return { + ...targets, + servers: (rows.payload || []).map((row) => { + const { url, headers, command, setup_command, env, all_bots, ...server } = parseMcpServerRow({ row }); + + return { + ...server, + access: { all_bots, ...access(server.id) }, + ...(targets.can_edit ? { url, headers, command, setup_command, env } : {}), + }; + }), + }; +} diff --git a/src/functions/backend/db/extensions/grab-run-envs.ts b/src/functions/backend/db/extensions/grab-run-envs.ts new file mode 100644 index 0000000..1957ea3 --- /dev/null +++ b/src/functions/backend/db/extensions/grab-run-envs.ts @@ -0,0 +1,31 @@ +import { EnvDict } from "@/src/dict/env-dict"; +import grabAllEnvs, { type DecryptedEnv } from "../envs/grab-all-envs"; + +type Params = { + /** + * The workspace the run is in: its envs beat general ones. + */ + workspace_id?: number | string | null; +}; + +/** + * The envs an MCP server's or custom tool's templates (`{{env.KEY}}`) read + * on one run, one per key: the workspace's beat general ones. App settings + * (`EnvDict`) never are. Unlike computers' envs, they don't have to be set + * on computers: the app fills them in itself. + */ +export default async function grabRunEnvs({ workspace_id }: Params = {}): Promise { + const app_keys = new Set(EnvDict.map((env) => env.key)); + const envs = (await grabAllEnvs()).filter((env) => !app_keys.has(env.key)); + const resolved = new Map(); + + for (const env of envs.filter((e) => e.paradigm == "general")) resolved.set(env.key, env); + + if (workspace_id) { + for (const env of envs.filter((e) => e.paradigm == "workspace" && e.workspace_id == Number(workspace_id))) { + resolved.set(env.key, env); + } + } + + return [...resolved.values()]; +} diff --git a/src/functions/backend/db/extensions/grab-tools-overview.ts b/src/functions/backend/db/extensions/grab-tools-overview.ts new file mode 100644 index 0000000..de3539d --- /dev/null +++ b/src/functions/backend/db/extensions/grab-tools-overview.ts @@ -0,0 +1,42 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_CUSTOM_TOOLS } from "@/db/types/db"; +import type { CustomToolsOverviewPayload } from "@/src/types"; +import grabCrewRole from "../access/grab-crew-role"; +import grabExtensionAccess from "./grab-extension-access"; +import grabExtensionTargets from "./grab-extension-targets"; +import parseCustomToolRow from "./parse-custom-tool-row"; + +type Params = { + user_id: number | string; +}; + +/** + * The Tools settings page: every custom tool with who gets it, and what it + * can be attached to. Its arguments and how it runs only go to the super + * admin. + */ +export default async function grabToolsOverview({ user_id }: Params): Promise { + const role = await grabCrewRole({ user_id }); + const [rows, access, targets] = await Promise.all([ + BunSQLite.sql({ sql: `SELECT * FROM custom_tools ORDER BY name ASC, id ASC` }), + grabExtensionAccess({ kind: "tool" }), + grabExtensionTargets({ role }), + ]); + + return { + ...targets, + tools: (rows.payload || []).map((row) => { + const tool = parseCustomToolRow({ row }); + + return { + id: tool.id, + name: tool.name, + description: tool.description, + kind: tool.kind, + enabled: tool.enabled, + access: { all_bots: tool.all_bots, ...access(tool.id) }, + ...(targets.can_edit ? { parameters: tool.parameters, config: tool.config } : {}), + }; + }), + }; +} diff --git a/src/functions/backend/db/extensions/parse-custom-tool-row.ts b/src/functions/backend/db/extensions/parse-custom-tool-row.ts new file mode 100644 index 0000000..ac08636 --- /dev/null +++ b/src/functions/backend/db/extensions/parse-custom-tool-row.ts @@ -0,0 +1,26 @@ +import type { BUN_SQLITE_COBBLE_CUSTOM_TOOLS } from "@/db/types/db"; +import type { CustomToolRecord } from "@/src/types"; +import parseJsonColumn from "./parse-json-column"; + +type Params = { + row: BUN_SQLITE_COBBLE_CUSTOM_TOOLS; +}; + +/** + * A `custom_tools` row with its JSON columns read. + */ +export default function parseCustomToolRow({ row }: Params): CustomToolRecord { + return { + id: Number(row.id), + name: row.name || "", + description: row.description || "", + parameters: parseJsonColumn>({ + text: row.parameters, + fallback: { type: "object", properties: {} }, + }), + all_bots: row.all_bots == 1, + enabled: row.enabled != 0, + kind: row.kind || "prompt", + config: parseJsonColumn>({ text: row.config, fallback: {} }), + } as CustomToolRecord; +} diff --git a/src/functions/backend/db/extensions/parse-json-column.ts b/src/functions/backend/db/extensions/parse-json-column.ts new file mode 100644 index 0000000..be31034 --- /dev/null +++ b/src/functions/backend/db/extensions/parse-json-column.ts @@ -0,0 +1,23 @@ +type Params = { + text?: string | null; + fallback: T; +}; + +/** + * Reads a JSON column of an MCP server or custom tool. The fallback when + * it's empty, not JSON, or not the same shape (array or object). + */ +export default function parseJsonColumn({ text, fallback }: Params): T { + if (!text) return fallback; + + try { + const value = JSON.parse(text); + + if (!value || typeof value != "object") return fallback; + if (Array.isArray(value) != Array.isArray(fallback)) return fallback; + + return value as T; + } catch { + return fallback; + } +} diff --git a/src/functions/backend/db/extensions/parse-mcp-server-row.ts b/src/functions/backend/db/extensions/parse-mcp-server-row.ts new file mode 100644 index 0000000..e7a5827 --- /dev/null +++ b/src/functions/backend/db/extensions/parse-mcp-server-row.ts @@ -0,0 +1,33 @@ +import type { BUN_SQLITE_COBBLE_MCP_SERVERS } from "@/db/types/db"; +import type { McpServerStatus, McpTransport } from "@/src/dict/mcp-dict"; +import type { McpServerRecord, McpToolInfo } from "@/src/types"; +import parseJsonColumn from "./parse-json-column"; + +type Params = { + row: BUN_SQLITE_COBBLE_MCP_SERVERS; +}; + +/** + * An `mcp_servers` row with its JSON columns read. + */ +export default function parseMcpServerRow({ row }: Params): McpServerRecord { + return { + id: Number(row.id), + name: row.name || "", + slug: row.slug || "", + description: row.description || "", + transport: (row.transport || "remote") as McpTransport, + url: row.url || "", + headers: parseJsonColumn>({ text: row.headers, fallback: {} }), + command: row.command || "", + setup_command: row.setup_command || "", + env: parseJsonColumn>({ text: row.env, fallback: {} }), + excluded_tools: parseJsonColumn({ text: row.excluded_tools, fallback: [] }), + tools: parseJsonColumn({ text: row.tools_cache, fallback: [] }), + all_bots: row.all_bots == 1, + enabled: row.enabled != 0, + status: (row.status || "untested") as McpServerStatus, + last_error: row.last_error || "", + last_synced_at: typeof row.last_synced_at == "number" ? row.last_synced_at : null, + }; +} diff --git a/src/functions/backend/db/extensions/save-custom-tool.ts b/src/functions/backend/db/extensions/save-custom-tool.ts new file mode 100644 index 0000000..727d0de --- /dev/null +++ b/src/functions/backend/db/extensions/save-custom-tool.ts @@ -0,0 +1,61 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_CUSTOM_TOOLS, BunSQLiteTables } from "@/db/types/db"; +import type { SaveCustomToolRequest } from "@/src/schemas/extension-schema"; +import AppError from "@/src/utils/app-error"; +import grabCustomTool from "./grab-custom-tool"; + +type Params = { + /** + * The super admin saving it (checked by the caller). + */ + user_id: number | string; + data: SaveCustomToolRequest; +}; + +/** + * Adds or edits a custom tool and returns its id. Names are unique. + */ +export default async function saveCustomTool({ user_id, data }: Params) { + const clash = ( + await BunSQLite.sql<{ id: number }>({ + sql: `SELECT id FROM custom_tools WHERE name = ? AND id != ?`, + values: [data.name, data.custom_tool_id || 0], + }) + ).payload?.[0]; + + if (clash) throw new AppError(`Another tool is already called ${data.name}.`); + + const fields = { + name: data.name, + description: data.description, + kind: data.kind, + parameters: JSON.stringify(data.parameters), + config: JSON.stringify(data.config), + enabled: data.enabled ? 1 : 0, + all_bots: data.all_bots ? 1 : 0, + } as const; + + if (data.custom_tool_id) { + const tool = await grabCustomTool({ custom_tool_id: data.custom_tool_id }); + const res = await BunSQLite.update({ + table: "custom_tools", + targetId: tool.id, + data: fields, + }); + + if (!res.success) throw new Error(`Couldn't save the tool: ${res.msg}`); + + return tool.id; + } + + const res = await BunSQLite.insert({ + table: "custom_tools", + data: [{ ...fields, user_id: Number(user_id) }], + }); + + const id = res.singleRes?.id || res.postInsertReturn?.insertId; + + if (!res.success || !id) throw new Error(`Couldn't add the tool: ${res.msg}`); + + return Number(id); +} diff --git a/src/functions/backend/db/extensions/save-mcp-server.ts b/src/functions/backend/db/extensions/save-mcp-server.ts new file mode 100644 index 0000000..65c0d4d --- /dev/null +++ b/src/functions/backend/db/extensions/save-mcp-server.ts @@ -0,0 +1,89 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import type { BUN_SQLITE_COBBLE_MCP_SERVERS, BunSQLiteTables } from "@/db/types/db"; +import { stopStdioMcpServer } from "@/src/harness/mcp/mcp-stdio-pool"; +import type { SaveMcpServerRequest } from "@/src/schemas/extension-schema"; +import mcpServerSlug from "@/src/utils/mcp-server-slug"; +import grabMcpServer from "./grab-mcp-server"; + +type Params = { + /** + * The super admin saving it (checked by the caller). + */ + user_id: number | string; + data: SaveMcpServerRequest; +}; + +/** + * Adds or edits an MCP server and returns its id. A new one gets a unique + * slug from its name (kept when it's renamed, so its tools keep their + * names). A running stdio server is stopped, so the next call starts it as + * saved. + */ +export default async function saveMcpServer({ user_id, data }: Params) { + const fields = { + name: data.name, + description: data.description, + transport: data.transport, + url: data.transport == "remote" ? data.url : "", + headers: JSON.stringify(data.transport == "remote" ? data.headers : {}), + command: data.transport == "stdio" ? data.command : "", + setup_command: data.transport == "stdio" ? data.setup_command : "", + env: JSON.stringify(data.transport == "stdio" ? data.env : {}), + excluded_tools: JSON.stringify(data.excluded_tools), + enabled: data.enabled ? 1 : 0, + all_bots: data.all_bots ? 1 : 0, + } as const; + + if (data.mcp_server_id) { + const server = await grabMcpServer({ mcp_server_id: data.mcp_server_id }); + + stopStdioMcpServer({ server_id: server.id }); + + const res = await BunSQLite.update({ + table: "mcp_servers", + targetId: server.id, + data: fields, + }); + + if (!res.success) throw new Error(`Couldn't save the MCP server: ${res.msg}`); + + return server.id; + } + + const res = await BunSQLite.insert({ + table: "mcp_servers", + data: [ + { + ...fields, + slug: await uniqueSlug({ name: data.name }), + status: "untested", + tools_cache: "[]", + user_id: Number(user_id), + }, + ], + }); + + const id = res.singleRes?.id || res.postInsertReturn?.insertId; + + if (!res.success || !id) throw new Error(`Couldn't add the MCP server: ${res.msg}`); + + return Number(id); +} + +async function uniqueSlug({ name }: { name: string }) { + const base = mcpServerSlug({ name }); + const taken = new Set( + ( + await BunSQLite.sql<{ slug: string }>({ + sql: `SELECT slug FROM mcp_servers WHERE slug = ? OR slug LIKE ?`, + values: [base, `${base}_%`], + }) + ).payload?.map((row) => row.slug), + ); + + if (!taken.has(base)) return base; + + for (let n = 2; ; n++) { + if (!taken.has(`${base}_${n}`)) return `${base}_${n}`; + } +} diff --git a/src/functions/backend/db/extensions/set-extension-access.ts b/src/functions/backend/db/extensions/set-extension-access.ts new file mode 100644 index 0000000..d14e2fd --- /dev/null +++ b/src/functions/backend/db/extensions/set-extension-access.ts @@ -0,0 +1,59 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import AppError from "@/src/utils/app-error"; +import assertCrewManager from "../access/assert-crew-manager"; +import { ExtensionTables, type ExtensionKind } from "./extension-tables"; + +type Params = { + user_id: number | string; + kind: ExtensionKind; + id: number; + bot_ids: number[]; + workspace_ids: number[]; + /** + * Super admins only: left alone for anyone else. + */ + all_bots?: boolean; +}; + +/** + * Attaches an MCP server or tool to exactly these bots and workspaces, for + * super admins and admins. An admin never sees the coordinator, so its + * attachment is kept as it is; only a super admin sets "all bots". + */ +export default async function setExtensionAccess({ user_id, kind, id, bot_ids, workspace_ids, all_bots }: Params) { + const role = await assertCrewManager({ user_id }); + const { table, bot_link, workspace_link, column, label } = ExtensionTables[kind]; + + const found = await BunSQLite.sql<{ id: number }>({ sql: `SELECT id FROM ${table} WHERE id = ?`, values: [id] }); + + if (!found.payload?.[0]) throw new AppError(`That ${label} doesn't exist.`); + + const coordinator = role != "super_admin" ? `AND bot_id NOT IN (SELECT id FROM bots WHERE kind = 'coordinator')` : ""; + const now = Date.now(); + + await BunSQLite.sql({ sql: `DELETE FROM ${bot_link} WHERE ${column} = ? ${coordinator}`, values: [id] }); + await BunSQLite.sql({ sql: `DELETE FROM ${workspace_link} WHERE ${column} = ?`, values: [id] }); + + for (const bot_id of new Set(bot_ids)) { + await BunSQLite.sql({ + sql: `INSERT OR IGNORE INTO ${bot_link} (created_at, updated_at, bot_id, ${column}) + SELECT ?, ?, id, ? FROM bots WHERE id = ? ${role != "super_admin" ? `AND kind = 'worker'` : ""}`, + values: [now, now, id, bot_id], + }); + } + + for (const workspace_id of new Set(workspace_ids)) { + await BunSQLite.sql({ + sql: `INSERT OR IGNORE INTO ${workspace_link} (created_at, updated_at, workspace_id, ${column}) + SELECT ?, ?, id, ? FROM workspaces WHERE id = ?`, + values: [now, now, id, workspace_id], + }); + } + + if (role == "super_admin" && all_bots !== undefined) { + await BunSQLite.sql({ + sql: `UPDATE ${table} SET all_bots = ?, updated_at = ? WHERE id = ?`, + values: [all_bots ? 1 : 0, now, id], + }); + } +} diff --git a/src/functions/backend/db/extensions/sync-mcp-server.ts b/src/functions/backend/db/extensions/sync-mcp-server.ts new file mode 100644 index 0000000..b70ba53 --- /dev/null +++ b/src/functions/backend/db/extensions/sync-mcp-server.ts @@ -0,0 +1,60 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import listMcpTools from "@/src/harness/mcp/list-mcp-tools"; +import openMcpServer from "@/src/harness/mcp/open-mcp-server"; +import runMcpSetup from "@/src/harness/mcp/run-mcp-setup"; +import grabMcpStdioEnv from "@/src/harness/mcp/grab-mcp-stdio-env"; +import { stopStdioMcpServer } from "@/src/harness/mcp/mcp-stdio-pool"; +import redactSecrets from "@/src/utils/redact-secrets"; +import grabMcpServer from "./grab-mcp-server"; +import grabRunEnvs from "./grab-run-envs"; + +type Params = { + mcp_server_id: number | string; + /** + * Runs a stdio server's setup again first, even if it already ran. + */ + reinstall?: boolean; +}; + +/** + * Connects to an MCP server (a stdio one is started, after its setup) and + * saves the tools it lists, which is what runs build their tools from. + * Saves the outcome on the server (`status`, `last_error`, secrets + * blanked out) and returns it; never throws for the server's own failure. + */ +export default async function syncMcpServer({ mcp_server_id, reinstall }: Params) { + const server = await grabMcpServer({ mcp_server_id }); + const envs = await grabRunEnvs(); + const open = openMcpServer({ server, envs }); + const now = Date.now(); + + try { + if (reinstall && server.transport == "stdio") { + stopStdioMcpServer({ server_id: server.id }); + await runMcpSetup({ server, env: grabMcpStdioEnv({ server, envs }), force: true }); + } + + const tools = await listMcpTools({ client: await open.client() }); + + await BunSQLite.sql({ + sql: `UPDATE mcp_servers SET tools_cache = ?, status = 'ok', last_error = '', last_synced_at = ?, updated_at = ? WHERE id = ?`, + values: [JSON.stringify(tools), now, now, server.id], + }); + + return { ok: true as const, tools }; + } catch (error: unknown) { + const message = redactSecrets({ + text: error instanceof Error ? error.message : String(error), + secrets: envs.filter((env) => env.secret), + }).slice(0, 4000); + + await BunSQLite.sql({ + sql: `UPDATE mcp_servers SET status = 'error', last_error = ?, last_synced_at = ?, updated_at = ? WHERE id = ?`, + values: [message, now, now, server.id], + }); + + return { ok: false as const, error: message }; + } finally { + await open.close(); + } +} diff --git a/src/functions/backend/db/extensions/test-custom-tool.ts b/src/functions/backend/db/extensions/test-custom-tool.ts new file mode 100644 index 0000000..6f1c141 --- /dev/null +++ b/src/functions/backend/db/extensions/test-custom-tool.ts @@ -0,0 +1,39 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { AppData } from "@/src/data/app-data"; +import buildCustomTool from "@/src/harness/tools/custom/build-custom-tool"; +import AppError from "@/src/utils/app-error"; +import grabCustomTool from "./grab-custom-tool"; +import grabRunEnvs from "./grab-run-envs"; + +type Params = { + custom_tool_id: number | string; + args: Record; + /** + * Where a script tool runs. The main computer when not given. + */ + computer_id?: number; +}; + +/** + * Runs a custom tool once with these arguments, as a bot would outside a + * workspace (general envs, the main computer for a script), and returns + * what the bot would read: its result, or `Error: ...`. + */ +export default async function testCustomTool({ custom_tool_id, args, computer_id }: Params) { + const tool = await grabCustomTool({ custom_tool_id }); + const main = + computer_id || + (await BunSQLite.sql<{ id: number }>({ sql: `SELECT id FROM computers WHERE kind = 'main' ORDER BY id LIMIT 1` })) + .payload?.[0]?.id; + + const harness_tool = buildCustomTool({ tool, envs: await grabRunEnvs(), computer_id: main }); + + if (!harness_tool) throw new AppError(`There's no computer to run this script on.`); + + try { + return await harness_tool.run({ args, signal: AbortSignal.timeout(AppData.CustomToolMaxTimeoutMs) }); + } catch (error: unknown) { + return `Error: ${error instanceof Error ? error.message : String(error)}`; + } +} + diff --git a/src/harness/mcp/grab-mcp-dir.ts b/src/harness/mcp/grab-mcp-dir.ts new file mode 100644 index 0000000..56494ba --- /dev/null +++ b/src/harness/mcp/grab-mcp-dir.ts @@ -0,0 +1,22 @@ +import { mkdir } from "fs/promises"; +import path from "path"; +import { AppData } from "@/src/data/app-data"; +import grabDirNames from "@/src/utils/grab-dir-names"; + +type Params = { + server_id: number | string; +}; + +/** + * Where a stdio MCP server runs and its setup installs: + * `DATA_DIR//`, created when missing. On the data + * volume in production, so installs survive app updates. + */ +export default async function grabMcpDir({ server_id }: Params) { + const { DATA_DIR } = grabDirNames(); + const dir = path.join(DATA_DIR, AppData.McpDataDir, String(server_id)); + + await mkdir(dir, { recursive: true }); + + return dir; +} diff --git a/src/harness/mcp/grab-mcp-stdio-env.ts b/src/harness/mcp/grab-mcp-stdio-env.ts new file mode 100644 index 0000000..5b66884 --- /dev/null +++ b/src/harness/mcp/grab-mcp-stdio-env.ts @@ -0,0 +1,23 @@ +import { getDefaultEnvironment } from "@modelcontextprotocol/sdk/client/stdio.js"; +import type { McpServerRecord } from "@/src/types"; +import fillTemplate from "@/src/utils/fill-template"; + +type Params = { + server: McpServerRecord; + envs: { key: string; value: string }[]; +}; + +/** + * The only environment a stdio MCP server (and its setup) gets: the + * basics a shell needs (`PATH`, `HOME`, `SHELL`, `USER`, ...) plus its own + * `env`, filled from the envs. Never the app's own environment, which + * holds its encryption key. + */ +export default function grabMcpStdioEnv({ server, envs }: Params): Record { + return { + ...getDefaultEnvironment(), + ...Object.fromEntries( + Object.entries(server.env).map(([key, value]) => [key, fillTemplate({ template: value, envs })]), + ), + }; +} diff --git a/src/harness/mcp/grab-mcp-tools.ts b/src/harness/mcp/grab-mcp-tools.ts index 013c5b7..9f91296 100644 --- a/src/harness/mcp/grab-mcp-tools.ts +++ b/src/harness/mcp/grab-mcp-tools.ts @@ -1,7 +1,8 @@ import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { AppData } from "@/src/data/app-data"; import type { HarnessTool } from "../types"; -import mcpResultToText, { type McpImage } from "./mcp-result-to-text"; +import listMcpTools from "./list-mcp-tools"; +import mcpHarnessTool from "./mcp-harness-tool"; +import type { McpImage } from "./mcp-result-to-text"; type Params = { client: Client; @@ -18,57 +19,15 @@ type Params = { }; /** - * Every tool an MCP server offers, as harness tools. Arguments go to the - * server as the model wrote them (the server checks them against its own - * schema); results come back as text via `mcpResultToText`. + * Every tool an MCP server offers, as harness tools (`mcpHarnessTool`). */ export default async function grabMcpTools({ client, exclude = [], describeImage, - redact = (text) => text, + redact, }: Params): Promise { - const tools: HarnessTool[] = []; - let cursor: string | undefined; - - do { - const page = await client.listTools(cursor ? { cursor } : undefined, { - timeout: AppData.McpConnectTimeoutMs, - }); - - for (const tool of page.tools) { - if (exclude.includes(tool.name)) continue; - - tools.push({ - name: tool.name, - description: tool.description || tool.title || tool.name, - parameters: tool.inputSchema as Record, - run: async ({ args, signal }) => { - try { - return redact( - await mcpResultToText({ - result: await client.callTool( - { name: tool.name, arguments: args }, - undefined, - { - signal, - timeout: AppData.McpToolTimeoutMs, - resetTimeoutOnProgress: true, - }, - ), - describeImage, - }), - ); - } catch (error: unknown) { - if (error instanceof Error) error.message = redact(error.message); - throw error; - } - }, - }); - } - - cursor = page.nextCursor; - } while (cursor); - - return tools; + return (await listMcpTools({ client })) + .filter((tool) => !exclude.includes(tool.name)) + .map((tool) => mcpHarnessTool({ tool, client, describeImage, redact })); } diff --git a/src/harness/mcp/host-shell.ts b/src/harness/mcp/host-shell.ts new file mode 100644 index 0000000..945fe16 --- /dev/null +++ b/src/harness/mcp/host-shell.ts @@ -0,0 +1,7 @@ +/** + * The host's default shell, which runs stdio MCP servers and their setup + * (` -c `). + */ +export default function hostShell() { + return process.env.SHELL || "/bin/bash"; +} diff --git a/src/harness/mcp/list-mcp-tools.ts b/src/harness/mcp/list-mcp-tools.ts new file mode 100644 index 0000000..698758c --- /dev/null +++ b/src/harness/mcp/list-mcp-tools.ts @@ -0,0 +1,33 @@ +import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { AppData } from "@/src/data/app-data"; +import type { McpToolInfo } from "@/src/types"; + +type Params = { + client: Client; +}; + +/** + * Every tool an MCP server offers, page by page. + */ +export default async function listMcpTools({ client }: Params): Promise { + const tools: McpToolInfo[] = []; + let cursor: string | undefined; + + do { + const page = await client.listTools(cursor ? { cursor } : undefined, { + timeout: AppData.McpConnectTimeoutMs, + }); + + for (const tool of page.tools) { + tools.push({ + name: tool.name, + description: tool.description || tool.title || tool.name, + input_schema: tool.inputSchema as Record, + }); + } + + cursor = page.nextCursor; + } while (cursor); + + return tools; +} diff --git a/src/harness/mcp/mcp-harness-tool.ts b/src/harness/mcp/mcp-harness-tool.ts new file mode 100644 index 0000000..befe079 --- /dev/null +++ b/src/harness/mcp/mcp-harness-tool.ts @@ -0,0 +1,59 @@ +import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { AppData } from "@/src/data/app-data"; +import type { HarnessTool } from "../types"; +import mcpResultToText, { type McpImage } from "./mcp-result-to-text"; + +type Params = { + /** + * What the model calls it. Defaults to the server's name for it. + */ + name?: string; + /** + * The tool as the server lists it. + */ + tool: { name: string; description?: string; input_schema: Record }; + /** + * The connected client, or how to get it on the first call. + */ + client: Client | (() => Promise); + describeImage?: (image: McpImage) => Promise; + redact?: (text: string) => string; +}; + +/** + * One MCP server tool as a harness tool. Arguments go to the server as the + * model wrote them (the server checks them against its own schema); + * results come back as text via `mcpResultToText`, redacted. + */ +export default function mcpHarnessTool({ + name, + tool, + client, + describeImage, + redact = (text) => text, +}: Params): HarnessTool { + return { + name: name || tool.name, + description: tool.description || tool.name, + parameters: tool.input_schema, + run: async ({ args, signal }) => { + try { + const connected = typeof client == "function" ? await client() : client; + + return redact( + await mcpResultToText({ + result: await connected.callTool({ name: tool.name, arguments: args }, undefined, { + signal, + timeout: AppData.McpToolTimeoutMs, + resetTimeoutOnProgress: true, + }), + describeImage, + }), + ); + } catch (error: unknown) { + if (error instanceof Error) error.message = redact(error.message); + throw error; + } + }, + }; +} diff --git a/src/harness/mcp/mcp-stdio-pool.ts b/src/harness/mcp/mcp-stdio-pool.ts new file mode 100644 index 0000000..6c40cd8 --- /dev/null +++ b/src/harness/mcp/mcp-stdio-pool.ts @@ -0,0 +1,140 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { AppData } from "@/src/data/app-data"; +import { SiteData } from "@/src/data/site-data"; +import type { McpServerRecord } from "@/src/types"; +import grabMcpDir from "./grab-mcp-dir"; +import grabMcpStdioEnv from "./grab-mcp-stdio-env"; +import hostShell from "./host-shell"; +import runMcpSetup from "./run-mcp-setup"; + +type Entry = { + /** + * The command, setup and environment it was started with: a change + * restarts it. + */ + key: string; + client: Promise; + transport?: StdioClientTransport; + timer?: ReturnType; +}; + +/** + * The stdio MCP servers running in this process, one per server, shared by + * every run. + */ +const pool = new Map(); + +type Params = { + server: McpServerRecord; + envs: { key: string; value: string }[]; +}; + +/** + * A connected client for a stdio MCP server, started on first use (after + * its setup, `runMcpSetup`) with the host's shell in its directory, and + * kept running for later calls until it's idle for `AppData.McpStdioIdleMs`, + * its configuration changes, or it exits. Its stderr's last lines explain + * a failed start. + */ +export default async function grabStdioMcpClient({ server, envs }: Params): Promise { + const env = grabMcpStdioEnv({ server, envs }); + const key = String(Bun.hash(JSON.stringify([server.command, server.setup_command, env]))); + const existing = pool.get(server.id); + + if (existing?.key == key) { + touch({ server_id: server.id, entry: existing }); + return existing.client; + } + + if (existing) stopStdioMcpServer({ server_id: server.id }); + + const entry = { key } as Entry; + + entry.client = start({ server, env, entry }).catch((error) => { + if (pool.get(server.id) == entry) pool.delete(server.id); + throw error; + }); + + pool.set(server.id, entry); + touch({ server_id: server.id, entry }); + + return entry.client; +} + +/** + * Stops a stdio MCP server running in this process, if it is (on a + * change, a delete, or idle). + */ +export function stopStdioMcpServer({ server_id }: { server_id: number }) { + const entry = pool.get(server_id); + + if (!entry) return; + + pool.delete(server_id); + clearTimeout(entry.timer); + entry.client.then((client) => client.close()).catch(() => {}); +} + +async function start({ server, env, entry }: { server: McpServerRecord; env: Record; entry: Entry }) { + await runMcpSetup({ server, env }); + + const transport = new StdioClientTransport({ + command: hostShell(), + args: ["-c", server.command], + cwd: await grabMcpDir({ server_id: server.id }), + env, + stderr: "pipe", + }); + let stderr = ""; + + transport.stderr?.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString()}`.slice(-4000); + }); + + entry.transport = transport; + + const client = new Client({ name: SiteData["SiteSlug"], version: "1.0.0" }); + + client.onclose = () => { + if (pool.get(server.id) == entry) { + pool.delete(server.id); + clearTimeout(entry.timer); + } + }; + + try { + await client.connect(transport, { timeout: AppData.McpConnectTimeoutMs * 4 }); + } catch (error) { + await client.close().catch(() => {}); + throw new Error( + `Couldn't start ${server.name}: ${error instanceof Error ? error.message : String(error)}${ + stderr.trim() ? `\n${stderr.trim().slice(-2000)}` : "" + }`, + ); + } + + return client; +} + +function touch({ server_id, entry }: { server_id: number; entry: Entry }) { + clearTimeout(entry.timer); + entry.timer = setTimeout(() => { + if (pool.get(server_id) == entry) stopStdioMcpServer({ server_id }); + }, AppData.McpStdioIdleMs); + entry.timer.unref?.(); +} + +/** + * Children go with the app. + */ +process.once("exit", () => { + for (const entry of pool.values()) { + const pid = entry.transport?.pid; + if (pid) { + try { + process.kill(pid); + } catch {} + } + } +}); diff --git a/src/harness/mcp/open-mcp-server.ts b/src/harness/mcp/open-mcp-server.ts new file mode 100644 index 0000000..4f93b4c --- /dev/null +++ b/src/harness/mcp/open-mcp-server.ts @@ -0,0 +1,48 @@ +import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import type { McpServerRecord } from "@/src/types"; +import fillTemplate from "@/src/utils/fill-template"; +import connectMcpServer from "./connect-mcp-server"; +import grabStdioMcpClient from "./mcp-stdio-pool"; + +type Params = { + server: McpServerRecord; + /** + * What its templates (`{{env.KEY}}`) read. + */ + envs: { key: string; value: string }[]; + signal?: AbortSignal; +}; + +/** + * How one run reaches an MCP server, without connecting yet: `client()` + * connects on the first call and is reused after. A remote server gets + * its own connection for the run (closed by `close`); a stdio server is + * shared through the process's pool and keeps running. + */ +export default function openMcpServer({ server, envs, signal }: Params) { + let remote: Promise | undefined; + + const client = () => { + if (server.transport == "stdio") return grabStdioMcpClient({ server, envs }); + + remote ||= connectMcpServer({ + url: fillTemplate({ template: server.url, envs }), + headers: Object.fromEntries( + Object.entries(server.headers).map(([key, value]) => [key, fillTemplate({ template: value, envs })]), + ), + signal, + }).catch((error) => { + remote = undefined; + throw error; + }); + + return remote; + }; + + const close = async () => { + if (!remote) return; + await (await remote.catch(() => null))?.close().catch(() => {}); + }; + + return { client, close }; +} diff --git a/src/harness/mcp/run-mcp-setup.ts b/src/harness/mcp/run-mcp-setup.ts new file mode 100644 index 0000000..c76e51b --- /dev/null +++ b/src/harness/mcp/run-mcp-setup.ts @@ -0,0 +1,64 @@ +import { readdir, rm, writeFile } from "fs/promises"; +import path from "path"; +import { AppData } from "@/src/data/app-data"; +import type { McpServerRecord } from "@/src/types"; +import grabMcpDir from "./grab-mcp-dir"; +import hostShell from "./host-shell"; + +type Params = { + server: McpServerRecord; + /** + * Its environment (`grabMcpStdioEnv`). + */ + env: Record; + /** + * Runs it even when this setup already ran. + */ + force?: boolean; +}; + +const MarkerPrefix = ".cobble-setup-"; + +/** + * Runs a stdio MCP server's setup command with the host's shell in its + * directory, once per setup command: a marker file named after it is left + * behind, so it runs again when the command changes, when `force` is set, + * or when the directory was cleared. The output goes to `setup.log`; a + * failure throws with its last lines. + */ +export default async function runMcpSetup({ server, env, force }: Params) { + const command = server.setup_command.trim(); + + if (!command) return; + + const dir = await grabMcpDir({ server_id: server.id }); + const marker = `${MarkerPrefix}${Bun.hash(command).toString(36)}`; + const existing = (await readdir(dir)).filter((file) => file.startsWith(MarkerPrefix)); + + if (!force && existing.includes(marker)) return; + + await Promise.all(existing.map((file) => rm(path.join(dir, file), { force: true }))); + + const proc = Bun.spawn([hostShell(), "-c", command], { + cwd: dir, + env, + stdout: "pipe", + stderr: "pipe", + timeout: AppData.McpSetupTimeoutMs, + }); + + const [stdout, stderr, exit_code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + const output = `${stdout}${stderr ? `\n${stderr}` : ""}`.trim(); + + await writeFile(path.join(dir, "setup.log"), `$ ${command}\n${output}\nExit code: ${exit_code}\n`); + + if (exit_code !== 0) { + throw new Error(`The setup command failed (exit code ${exit_code}): ${output.slice(-2000) || "no output"}`); + } + + await writeFile(path.join(dir, marker), command); +} diff --git a/src/harness/tools/custom/build-custom-tool.ts b/src/harness/tools/custom/build-custom-tool.ts new file mode 100644 index 0000000..90c02ad --- /dev/null +++ b/src/harness/tools/custom/build-custom-tool.ts @@ -0,0 +1,70 @@ +import type { CustomToolRecord } from "@/src/types"; +import redactSecrets from "@/src/utils/redact-secrets"; +import type { HarnessTool } from "../../types"; +import checkRequiredArgs from "./check-required-args"; +import runHttpTool from "./run-http-tool"; +import runPromptTool from "./run-prompt-tool"; +import runScriptTool from "./run-script-tool"; + +type Params = { + tool: CustomToolRecord; + /** + * What `{{env.KEY}}` reads (`grabRunEnvs`). Secret values are blanked + * out of every result and error. + */ + envs: { key: string; value: string; secret: boolean }[]; + /** + * Where a script tool runs. Without one, a script tool isn't offered. + */ + computer_id?: number | string | null; + bot_id?: number | string; + workspace_id?: number | string | null; +}; + +/** + * A custom tool as a harness tool: its name, description and JSON schema + * as the model sees them, and `run` dispatching on its kind (an HTTP + * request, a script on the computer, or a prompt). Null for a script tool + * when the run has no computer. + */ +export default function buildCustomTool({ + tool, + envs, + computer_id, + bot_id, + workspace_id, +}: Params): HarnessTool> | null { + if (tool.kind == "script" && !computer_id) return null; + + const secrets = envs.filter((env) => env.secret); + const redact = (text: string) => redactSecrets({ text, secrets }); + + const call = ({ args, signal }: { args: Record; signal: AbortSignal }) => { + switch (tool.kind) { + case "http": + return runHttpTool({ config: tool.config, args, envs, signal }); + case "script": + return runScriptTool({ config: tool.config, args, computer_id: computer_id!, workspace_id, signal }); + case "prompt": + return runPromptTool({ config: tool.config, args, envs, bot_id, workspace_id, signal }); + } + }; + + return { + name: tool.name, + description: + tool.kind == "script" + ? `${tool.description}\n(Runs on your current computer.)` + : tool.description, + parameters: tool.parameters, + run: async ({ args, signal }) => { + try { + checkRequiredArgs({ parameters: tool.parameters, args }); + return redact(await call({ args, signal })); + } catch (error: unknown) { + if (error instanceof Error) error.message = redact(error.message); + throw error; + } + }, + }; +} diff --git a/src/harness/tools/custom/check-required-args.ts b/src/harness/tools/custom/check-required-args.ts new file mode 100644 index 0000000..931079e --- /dev/null +++ b/src/harness/tools/custom/check-required-args.ts @@ -0,0 +1,21 @@ +type Params = { + /** + * The tool's JSON schema. + */ + parameters: Record; + args: Record; +}; + +/** + * Throws when the model left out an argument the tool's schema requires. + * The rest of the schema is the model's to follow: custom tools fill + * templates, so a wrong type does no harm the call wouldn't show. + */ +export default function checkRequiredArgs({ parameters, args }: Params) { + const required = Array.isArray(parameters.required) ? parameters.required.map(String) : []; + const missing = required.filter((key) => args[key] === undefined || args[key] === null || args[key] === ""); + + if (missing.length) { + throw new Error(`Missing required arguments: ${missing.join(", ")}.`); + } +} diff --git a/src/harness/tools/custom/run-http-tool.ts b/src/harness/tools/custom/run-http-tool.ts new file mode 100644 index 0000000..2513c53 --- /dev/null +++ b/src/harness/tools/custom/run-http-tool.ts @@ -0,0 +1,58 @@ +import { AppData } from "@/src/data/app-data"; +import type { CustomToolHttpConfig } from "@/src/types"; +import fillTemplate from "@/src/utils/fill-template"; + +type Params = { + config: CustomToolHttpConfig; + args: Record; + envs: { key: string; value: string }[]; + signal: AbortSignal; +}; + +/** + * Calls the tool's API from the app: URL (arguments URL-encoded), headers + * and body filled in from the arguments and envs. With no body template, + * anything but GET sends the arguments as JSON. Returns the response (JSON + * pretty-printed); a non-2xx answer throws with its status and body. + */ +export default async function runHttpTool({ config, args, envs, signal }: Params) { + const fill = (template: string, encode?: (value: string) => string) => + fillTemplate({ template, args, envs, encode }); + + const method = config.method || "GET"; + const headers = new Headers( + Object.entries(config.headers || {}).map(([key, value]) => [key, fill(value)] as [string, string]), + ); + const body = + method == "GET" ? undefined : config.body?.trim() ? fill(config.body) : JSON.stringify(args); + + if (body && !headers.has("content-type")) headers.set("content-type", "application/json"); + + const res = await fetch(fill(config.url, encodeURIComponent), { + method, + headers, + body, + signal: AbortSignal.any([ + signal, + AbortSignal.timeout( + Math.min(config.timeout_ms || AppData.CustomToolDefaultTimeoutMs, AppData.CustomToolMaxTimeoutMs), + ), + ]), + }); + + const text = prettify({ text: await res.text() }); + + if (!res.ok) { + throw new Error(`HTTP ${res.status} ${res.statusText}${text ? `: ${text.slice(0, 4000)}` : ""}`); + } + + return text || `HTTP ${res.status}, no body.`; +} + +function prettify({ text }: { text: string }) { + try { + return JSON.stringify(JSON.parse(text), null, 2); + } catch { + return text.trim(); + } +} diff --git a/src/harness/tools/custom/run-prompt-tool.ts b/src/harness/tools/custom/run-prompt-tool.ts new file mode 100644 index 0000000..6d576f2 --- /dev/null +++ b/src/harness/tools/custom/run-prompt-tool.ts @@ -0,0 +1,30 @@ +import type { CustomToolPromptConfig } from "@/src/types"; +import fillTemplate from "@/src/utils/fill-template"; +import generateText from "../../text/generate-text"; + +type Params = { + config: CustomToolPromptConfig; + args: Record; + envs: { key: string; value: string }[]; + /** + * Its models are used, like the run's. + */ + bot_id?: number | string; + workspace_id?: number | string | null; + signal: AbortSignal; +}; + +/** + * Fills the tool's prompt in and answers it with one text model turn (no + * tools). Returns the answer. + */ +export default async function runPromptTool({ config, args, envs, bot_id, workspace_id, signal }: Params) { + const result = await generateText({ + messages: [{ role: "user", content: fillTemplate({ template: config.prompt || "", args, envs }) }], + bot_id, + workspace_id, + signal, + }); + + return result.text || "(no answer)"; +} diff --git a/src/harness/tools/custom/run-script-tool.ts b/src/harness/tools/custom/run-script-tool.ts new file mode 100644 index 0000000..240f5d8 --- /dev/null +++ b/src/harness/tools/custom/run-script-tool.ts @@ -0,0 +1,73 @@ +import { AppData } from "@/src/data/app-data"; +import { CustomToolScriptLanguages } from "@/src/dict/custom-tool-dict"; +import grabComputerMcpUrl from "@/src/functions/backend/db/computers/grab-computer-mcp-url"; +import grabComputerEnvs from "@/src/functions/backend/db/envs/grab-computer-envs"; +import type { CustomToolScriptConfig } from "@/src/types"; +import computerNotReadyMessage from "@/src/utils/computer-not-ready-message"; +import redactSecrets from "@/src/utils/redact-secrets"; +import runComputerCommand from "../computer/run-computer-command"; +import writeComputerEnv from "../computer/write-computer-env"; + +type Params = { + config: CustomToolScriptConfig; + args: Record; + computer_id: number | string; + /** + * The workspace the run is in: its envs apply on the computer. + */ + workspace_id?: number | string | null; + signal: AbortSignal; +}; + +/** + * Runs the tool's code on a computer over the sandbox's REST API, with the + * computer's envs set (`writeComputerEnv`) and the arguments as JSON in + * `COBBLE_ARGS` and on stdin. Code and arguments travel base64-encoded, + * so nothing in them needs quoting. Returns what it printed (stdout and + * stderr), secrets redacted; a non-zero exit throws with it. + */ +export default async function runScriptTool({ config, args, computer_id, workspace_id, signal }: Params) { + const target = await grabComputerMcpUrl({ computer_id }); + + if (!target) throw new Error(`Computer #${computer_id} has no container to run on.`); + if (!target.ready) throw new Error(computerNotReadyMessage(target)); + + const envs = await grabComputerEnvs({ computer_id, workspace_id }); + const { shell_path } = await writeComputerEnv({ base_url: target.base_url, envs, signal }); + const language = + CustomToolScriptLanguages.find((l) => l.value == config.language) || CustomToolScriptLanguages[0]; + + const code = Buffer.from(config.code || "").toString("base64"); + const input = Buffer.from(JSON.stringify(args)).toString("base64"); + + const result = await runComputerCommand({ + base_url: target.base_url, + /** + * In a subshell: `exit` in the sandbox's own shell would end its + * session. + */ + command: `(${[ + `f=$(mktemp --suffix=.${language.extension})`, + `echo ${code} | base64 -d > "$f"`, + `[ -f ${shell_path} ] && . ${shell_path}`, + `export COBBLE_ARGS="$(echo ${input} | base64 -d)"`, + `printf '%s' "$COBBLE_ARGS" | ${language.interpreter} "$f" 2>&1`, + `status=$?`, + `rm -f "$f"`, + `exit $status`, + ].join("; ")})`, + timeout_ms: Math.min(config.timeout_ms || AppData.CustomToolDefaultTimeoutMs, AppData.CustomToolMaxTimeoutMs), + signal, + }); + + const output = redactSecrets({ + text: result.output.trim(), + secrets: envs.filter((env) => env.secret), + }); + + if (result.exit_code) { + throw new Error(`Exit code ${result.exit_code}${output ? `:\n${output}` : "."}`); + } + + return output || "(no output)"; +} diff --git a/src/harness/tools/extensions/open-bot-extensions.ts b/src/harness/tools/extensions/open-bot-extensions.ts new file mode 100644 index 0000000..a4b783c --- /dev/null +++ b/src/harness/tools/extensions/open-bot-extensions.ts @@ -0,0 +1,108 @@ +import grabBotExtensions from "@/src/functions/backend/db/extensions/grab-bot-extensions"; +import grabRunEnvs from "@/src/functions/backend/db/extensions/grab-run-envs"; +import mcpToolName from "@/src/utils/mcp-tool-name"; +import redactSecrets from "@/src/utils/redact-secrets"; +import mcpHarnessTool from "../../mcp/mcp-harness-tool"; +import openMcpServer from "../../mcp/open-mcp-server"; +import readImage from "../../media/read-image"; +import type { HarnessTool } from "../../types"; +import buildCustomTool from "../custom/build-custom-tool"; + +type Params = { + bot_id: number | string; + workspace_id?: number | string | null; + /** + * Where script tools run: the task's computer, or in chat the bot's + * current one. None leaves script tools out. + */ + computer_id?: number | string | null; + /** + * Names the run's built-in tools already use: they win. + */ + taken?: string[]; + signal?: AbortSignal; +}; + +export type OpenBotExtensions = { + tools: HarnessTool[]; + /** + * For the system prompt, only when something's wrong (an MCP server + * with no tools listed), so the bot can say so. Empty otherwise. + */ + note: string; + close: () => Promise; +}; + +/** + * The MCP servers' and custom tools' tools a bot gets on one run + * (`grabBotExtensions`), as harness tools. MCP tools are built from each + * server's last listing (`__`) and connect on their first call, + * so a run that doesn't use them never waits on them. Secret env values are + * blanked out of every result. Call `close` when the run ends. + */ +export default async function openBotExtensions({ + bot_id, + workspace_id, + computer_id, + taken = [], + signal, +}: Params): Promise { + const { servers, tools } = await grabBotExtensions({ bot_id, workspace_id }); + + if (!servers.length && !tools.length) return { tools: [], note: "", close: async () => {} }; + + const envs = await grabRunEnvs({ workspace_id }); + const secrets = envs.filter((env) => env.secret); + const redact = (text: string) => redactSecrets({ text, secrets }); + const names = new Set(taken); + const result: HarnessTool[] = []; + const notes: string[] = []; + const opened: ReturnType[] = []; + + const add = (tool: HarnessTool | null) => { + if (!tool || names.has(tool.name)) return; + names.add(tool.name); + result.push(tool); + }; + + for (const tool of tools) { + add(buildCustomTool({ tool, envs, computer_id, bot_id, workspace_id })); + } + + for (const server of servers) { + const listed = server.tools.filter((tool) => !server.excluded_tools.includes(tool.name)); + + if (!listed.length) { + notes.push( + `- ${server.name}: ${server.status == "error" ? `unavailable (${redact(server.last_error).slice(0, 300)})` : "no tools listed yet"}.`, + ); + continue; + } + + const open = openMcpServer({ server, envs, signal }); + opened.push(open); + + for (const tool of listed) { + add( + mcpHarnessTool({ + name: mcpToolName({ slug: server.slug, name: tool.name }), + tool: { ...tool, description: `${server.name}: ${tool.description}` }, + client: open.client, + redact, + describeImage: ({ data, mime_type }) => + readImage({ url: `data:${mime_type};base64,${data}`, signal, bot_id, workspace_id }), + }), + ); + } + } + + return { + tools: result, + note: notes.length + ? `These MCP servers were set up for you but you can't use them right now; if the user needs them, say so (an admin fixes them in Settings, under MCPs):\n${notes.join("\n")}` + : "", + close: async () => { + await Promise.all(opened.map((open) => open.close())); + }, + }; +} diff --git a/src/layouts/bots/(partials)/add-workspace-form.tsx b/src/layouts/bots/(partials)/add-workspace-form.tsx index 8bd3ea2..80cf6df 100644 --- a/src/layouts/bots/(partials)/add-workspace-form.tsx +++ b/src/layouts/bots/(partials)/add-workspace-form.tsx @@ -1,7 +1,7 @@ import { useState } from "react"; import Form from "@moduletrace/twui/form/Form"; import Input from "@moduletrace/twui/form/Input"; -import Textarea from "@moduletrace/twui/form/Textarea"; +import CodeEditor from "@/src/components/general/code-editor"; import ImagePickField from "@/src/components/general/image-pick-field"; import WorkspaceAvatar from "@/src/components/general/workspace-avatar"; import setProfileImage from "@/src/functions/frontend/set-profile-image"; @@ -82,16 +82,14 @@ export default function AddWorkspaceForm({ onCancel }: Props) { defaultValue={description} onChange={(e) => setDescription(e.target.value)} /> -