From 56daf0c462599ca72af7dc53a4973e53b646e652 Mon Sep 17 00:00:00 2001 From: Benjamin Toby Date: Mon, 19 Oct 2026 05:35:44 +0100 Subject: [PATCH] Open computers from the chat, let bots ask for a take-over, show task progress - Serve each computer to its owner under /computers// (HTTP and websockets proxied to the AIO sandbox) and show it in a shared ComputerView with browser, terminal and code modes, opened from the computers modal or a take-over button in the thread. - ask_user_to_take_over lets a bot hand a computer to the user for a login or captcha; the message gets a take-over button and needs-you, and the viewer hands it back with a chat message. - While a task runs the thread shows " is working..." with the task's latest step; steps read as plain words, not tool names. - A coordinator with no workers no longer claims to delegate, and bots describe started tasks in the first person. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 10 +- src/bots-queue/describe-job-step.test.ts | 32 +++++- src/bots-queue/describe-job-step.ts | 38 ++++--- src/bots-queue/run-bot-reply.ts | 34 +++++-- src/bots-queue/run-bot-task.ts | 31 +++++- src/bots-queue/with-take-over-refs.test.ts | 24 +++++ src/bots-queue/with-take-over-refs.ts | 21 ++++ .../general/computer-view-frame.tsx | 29 ++++++ .../general/computer-view-mode-button.tsx | 31 ++++++ src/components/general/computer-view.tsx | 77 +++++++++++++++ src/data/harness-prompts.ts | 11 ++- src/dict/bot-queue-dict.ts | 26 +++++ src/dict/computer-dict.ts | 31 ++++++ .../server/computers/computer-proxy-socket.ts | 69 +++++++++++++ .../server/computers/computer-proxy-target.ts | 46 +++++++++ .../server/computers/handle-computer-proxy.ts | 70 +++++++++++++ src/harness/tools/bot/grab-bot-chat-tools.ts | 16 ++- src/harness/tools/bot/start-task-tool.ts | 2 +- .../tools/computer/ask-to-take-over-tool.ts | 65 ++++++++++++ .../[bot_id]/(data)/bot-computers-context.ts | 6 +- .../[bot_id]/(hooks)/use-bot-computers.ts | 28 +++++- .../[bot_id]/(hooks)/use-task-progress.ts | 54 ++++++++++ .../(partials)/bot-computer-option.tsx | 32 ++++-- .../(partials)/bot-computers-modal.tsx | 11 ++- .../[bot_id]/(partials)/bot-thinking-row.tsx | 38 +++++-- .../(partials)/computer-viewer-modal.tsx | 98 +++++++++++++++++++ .../[bot_id]/(partials)/markdown-link.tsx | 22 ++++- .../[bot_id]/(partials)/message-thread.tsx | 11 ++- .../[bot_id]/(partials)/take-over-chip.tsx | 43 ++++++++ .../bots/[bot_id]/(sections)/conversation.tsx | 32 +++++- src/pages/admin/bots/[bot_id]/index.server.ts | 5 +- src/server.ts | 6 ++ src/types/index.ts | 4 + src/utils/computer-proxy-headers.test.ts | 68 +++++++++++++ src/utils/computer-proxy-request-headers.ts | 49 ++++++++++ src/utils/computer-proxy-response-headers.ts | 53 ++++++++++ src/utils/computer-view-url.ts | 20 ++++ src/utils/link-computer-refs.test.ts | 9 ++ src/utils/link-computer-refs.ts | 26 +++-- src/utils/parse-computer-proxy-path.test.ts | 24 +++++ src/utils/parse-computer-proxy-path.ts | 22 +++++ src/utils/take-over-ref.ts | 15 +++ src/websocket/index.ts | 7 ++ src/websocket/types.ts | 18 +++- 44 files changed, 1289 insertions(+), 75 deletions(-) create mode 100644 src/bots-queue/with-take-over-refs.test.ts create mode 100644 src/bots-queue/with-take-over-refs.ts create mode 100644 src/components/general/computer-view-frame.tsx create mode 100644 src/components/general/computer-view-mode-button.tsx create mode 100644 src/components/general/computer-view.tsx create mode 100644 src/functions/server/computers/computer-proxy-socket.ts create mode 100644 src/functions/server/computers/computer-proxy-target.ts create mode 100644 src/functions/server/computers/handle-computer-proxy.ts create mode 100644 src/harness/tools/computer/ask-to-take-over-tool.ts create mode 100644 src/pages/admin/bots/[bot_id]/(hooks)/use-task-progress.ts create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/computer-viewer-modal.tsx create mode 100644 src/pages/admin/bots/[bot_id]/(partials)/take-over-chip.tsx create mode 100644 src/utils/computer-proxy-headers.test.ts create mode 100644 src/utils/computer-proxy-request-headers.ts create mode 100644 src/utils/computer-proxy-response-headers.ts create mode 100644 src/utils/computer-view-url.ts create mode 100644 src/utils/parse-computer-proxy-path.test.ts create mode 100644 src/utils/parse-computer-proxy-path.ts create mode 100644 src/utils/take-over-ref.ts diff --git a/AGENTS.md b/AGENTS.md index 50b5a8c..acce313 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,15 +88,17 @@ Inspect live data with `bun llm-code-assistant/query-db.ts ""` - a read-onl - **Routes:** `/admin/bots` ensures the coordinator and redirects to its chat (`/admin/bots/1` on a fresh install); `/admin/bots/[bot_id]` renders the conversation (header, thread and composer, the last two held to 760px; there is no right panel); `/admin/bots/[bot_id]/models` is a worker bot's own model settings (the coordinator's redirects to its chat). Both render in the full-bleed `src/layouts/bots/` layout (rail of conversations + user popover), which `__root.tsx` uses for every `/admin/*` page. - **Server:** `src/functions/backend/db/bots/` (`grab-bots`, `grab-bot-messages`, `grab-active-bot-task`, `grab-user-bot`, `mark-bot-read`, `insert-user-message`). `api/admin/bots/send-message.ts` posts a user message to a bot owned by the session user (zod: `src/schemas/bot-schema.ts`), queues the bot's reply (`enqueueBotReply`) and wakes the queue (`src/bots-queue/wake-bots-queue.ts`; never import `src/bots-queue/index.ts` from a page or API route, it pulls the bunext runtime into the page bundles). `upload-attachment.ts` stores one file for the next message (private `media` row, paradigm `bot-attachment`, type from `attachmentFileKind`, documents get `text_content`, limits in `AppData.BotAttachment*`) and `send-message` takes their `media_ids` (checked to be the user's own by `grabAttachableMedia`), linked in `bot_message_media`; messages carry `attachments` (`BotMessageWithAttachments`). `mark-read.ts` clears unread for a message that arrives in the open conversation; `save-harness-settings.ts` saves a worker bot's models. - **Message rendering:** bot messages are markdown, rendered with `react-markdown` + `remark-gfm` (raw HTML never rendered) through the twui-based map in `[bot_id]/(data)/message-markdown-components.tsx` (`markdown-link`, `markdown-list`, `markdown-code-block` partials). User messages stay plain text. -- **Attachments and thinking:** the composer uploads files as they're picked, pasted or dropped (`use-composer-attachments`), shows them as chips, and sends once none is still uploading; the thread shows images as thumbnails and other files as chips. Replies read the files of the last `BotReplyAttachmentMessages` messages with any (`load-attachment-parts.ts`: images and audio as parts, documents as their text up to `BotAttachmentMaxTextChars`, videos as a note); older ones are only named. Tasks don't get the files yet. While a reply runs (face `working`, status `BotReplyJobTitle`) or waits for a model (`BotWaitingStatusLine`), `BotThinkingRow` sits under the last message. Live notices come from the process that ran the job at once, and from every other process through the live relay. +- **Attachments and thinking:** the composer uploads files as they're picked, pasted or dropped (`use-composer-attachments`), shows them as chips, and sends once none is still uploading; the thread shows images as thumbnails and other files as chips. Replies read the files of the last `BotReplyAttachmentMessages` messages with any (`load-attachment-parts.ts`: images and audio as parts, documents as their text up to `BotAttachmentMaxTextChars`, videos as a note); older ones are only named. Tasks don't get the files yet. While a reply runs (face `working`, status `BotReplyJobTitle`) or waits for a model (`BotWaitingStatusLine`), `BotThinkingRow` sits under the last message; while a task runs it reads " is working…" with the task's latest step. A coordinator with no workers gets `CoordinatorSoloNote` instead of `CoordinatorReplyNote`, so it never claims to have handed work to a worker. Live notices come from the process that ran the job at once, and from every other process through the live relay. - **Avatars:** `BotStone` (`src/components/general/bot-stone.tsx`) renders the stone inline from `src/data/stone-geometry.ts`, hue as a fill token, face (`idle|working|done|asleep|needs-you`) drawn as eyes. The coordinator is hue-less (ink body, paper eyes; inverts in dark). Never put a stone in a circle/badge or add outline/shadow. Wordmark is vector (`Wordmark`, `src/data/wordmark-paths.ts`), never live text. - **Copy helpers:** `src/utils/crew-status.ts` (`3 working · 1 needs you`, "Three bots are working. One needs you."), `src/utils/relative-time.ts` (relative timestamps only). - **Prompt and memory (self-learning):** a bot has `system_prompt` (core instructions) and a separate `memory` (markdown, what it must always remember). The bot rewrites both as it adapts to the user (`update-bot-system-prompt`, `update-bot-memory` in `src/functions/backend/db/bots/`; whole-text replace, capped at `AppData.PromptMaxLength`). `grab-bot-prompt` builds the full prompt via `composeBotPrompt` (`src/utils/compose-bot-prompt.ts`): core instructions, then each non-archived workspace's `system_prompt` under `# Workspace: `, then `# Memory`. -- **Queue:** every bot job goes through `bots_queue`, with `kind` (`BotJobKinds`): `reply` (answers the user in chat, no task row) or `task` (long-running work, typically on a computer). Long-running work goes through `bots_queue`, a bun-sqlite `isQueue` table worked with the native `BunSQLite.queue` API (`status` pending|processing|completed|failed, `attempts`, `available_at`, `locked_at`/`locked_by`, `last_error`; options in `AppData.BotQueue*`). Not every message needs a job: the bot replies right away that it's on it and `enqueueBotJob` queues the brief (`instructions`, `params`, optional `message_id`/`computer_id`, the computer checked against the bot's access; a `bot_tasks` row is created unless `task_id` is given). DB helpers in `src/functions/backend/db/bots-queue/`: `claimBotJobs`, `updateBotJob` (renews the lock, saves `progress`/`checkpoint`; false once lost or cancelled), `completeBotJob` (result message, task done, face done/needs-you), `failBotJob` (backoff retry, or `fatal`; out of attempts posts a needs-you message via `notifyBotJobFailed`), `cancelBotJob` (failed with `BotJobCancelledError`, `src/dict/bot-queue-dict.ts`), `releaseWorkerBotJobs`. Every model turn of a job is a step in `bots_queue_updates` (`kind` tool|note, one-line `text`, JSON `metadata`; tool names, never arguments; `describe-job-step.ts`), written through the job context's `report` (the text loop's `onStep`), which also sets `progress` and pushes `server:bot-job-update` (`BotJobUpdateData`). Steps are deleted with their job when it's pruned. The active task's latest `AppData.BotJobUpdatesShown` steps ride on `BotTaskWithSteps.updates` (`grabBotJobUpdates`); nothing in the UI shows them since the working-on panel was removed. +- **Queue:** every bot job goes through `bots_queue`, with `kind` (`BotJobKinds`): `reply` (answers the user in chat, no task row) or `task` (long-running work, typically on a computer). Long-running work goes through `bots_queue`, a bun-sqlite `isQueue` table worked with the native `BunSQLite.queue` API (`status` pending|processing|completed|failed, `attempts`, `available_at`, `locked_at`/`locked_by`, `last_error`; options in `AppData.BotQueue*`). Not every message needs a job: the bot replies right away that it's on it and `enqueueBotJob` queues the brief (`instructions`, `params`, optional `message_id`/`computer_id`, the computer checked against the bot's access; a `bot_tasks` row is created unless `task_id` is given). DB helpers in `src/functions/backend/db/bots-queue/`: `claimBotJobs`, `updateBotJob` (renews the lock, saves `progress`/`checkpoint`; false once lost or cancelled), `completeBotJob` (result message, task done, face done/needs-you), `failBotJob` (backoff retry, or `fatal`; out of attempts posts a needs-you message via `notifyBotJobFailed`), `cancelBotJob` (failed with `BotJobCancelledError`, `src/dict/bot-queue-dict.ts`), `releaseWorkerBotJobs`. Every model turn of a job is a step in `bots_queue_updates` (`kind` tool|note, one-line `text`, JSON `metadata`; tool names, never arguments; `describe-job-step.ts`), written through the job context's `report` (the text loop's `onStep`), which also sets `progress` and pushes `server:bot-job-update` (`BotJobUpdateData`). Steps are deleted with their job when it's pruned. The active task's latest `AppData.BotJobUpdatesShown` steps ride on `BotTaskWithSteps.updates` (`grabBotJobUpdates`); the conversation (`bot_task` page prop, `use-task-progress`) shows the latest one under `BotThinkingRow`'s " is working…" while a task runs. Step text is the first line the model wrote, else plain labels for its tools (`JobStepToolLabels` in `src/dict/bot-queue-dict.ts`). - **Queue handler:** `src/bots-queue/index.ts`, started from `server.ts` after cron and gated the same way. It releases its own cut-off jobs on startup (stable worker `AppData.BotQueueWorker`), then polls every `BotQueuePollMs`: reports jobs bun-sqlite failed on a lock timeout (`report-lost-bot-jobs`), claims per kind (`claimBotJobs` mirrors bun-sqlite's atomic claim plus `kind`, since `queue.claim` has no filter) up to `BotQueueReplyConcurrency` replies and `BotQueueConcurrency` tasks on separate slots, and runs each with `processBotJob` (heartbeat at a quarter of the lock timeout, abort on cancel/loss, one task per computer and one reply per bot via `createKeyLock`, face `working` once it starts, outcome pushed live). The actual work is `run-bot-job.ts` (`BotJobRunner`, context in `types.ts`), which dispatches to `run-bot-reply.ts` and `run-bot-task.ts`. Throw `BotJobFatalError` for failures retrying can't fix. When every model is resting (`HarnessError.resting`, nothing was tried) the job is put back with `deferBotJob` until the first comes back, its attempt refunded and the bot's face `asleep`; other retryable harness errors retry no sooner than their `retry_at`. The `prune-bots-queue` cron deletes finished jobs after `BotQueueRetentionDays`. - **Live updates:** server push only. `sendToUser` (`src/websocket/(functions)/send-to-user.ts`) sends to one user's sockets; `pushBotNotice` sends `server:bot-message` (the message row, user messages too, for other tabs) and `server:bot-updated` (`BotUpdatedData`: face, status line, active task). The bots layout opens one socket (twui `useWebSocket`); components listen with `useServerEvent` (`src/hooks/`), and `useLiveBots` keeps the rail and header current. The conversation dedupes messages by id. Every process also runs `liveRelay` (`src/live-relay/`): every `AppData.LiveRelayPollMs` it reads new `bot_messages`, new `bots_queue_updates` and changed bots from the database and pushes them to its own sockets, so work done by another process still shows live; clients drop what they already have. `server:bot-updated` carries the bot's `computer_id` when known, which keeps the header's computer in step across tabs. A rail row's subtitle is the bot's status line while it has one, else (coordinator) the crew counts while a worker is busy or needs you, else its last message as plain text (`src/utils/message-preview.ts`). - **Workspaces:** `workspaces` (per user, `name`, `description`, `system_prompt` shared by every bot in it, `archived`), with bots assigned through `workspace_bots` and computers through `workspace_computers`. Helpers in `src/functions/backend/db/workspaces/` (`create-workspace`, `update-workspace`, `add/remove-bot-*`, `add/remove-computer-*`), all ownership-checked against the session user. - **Computers:** a computer is an AIO sandbox in its own docker container on the `AppData.DockerNetworkName` network. `containers` records every docker container the app knows about except the main app container (`name` doubles as the hostname, plus `docker_id`, `image`, `network`, `port`, `status`, `metadata`). `computers` (per user, `kind` main|sandbox, `container_id`) and `bot_computers` (direct access). Each user has one main computer, recorded with the account (signup, `bun run seed`) and as a fallback by `/admin/bots` via `ensureMainComputer` (`src/functions/backend/db/computers/ensure-main-computer.ts`, idempotent), which also records its container (`AppData.MainComputerContainerName`, created by the installer) and grants the coordinator access. That one grant can't be removed (`isLockedComputerAccess`, enforced in `remove-bot-from-computer` and the `bot_computers` CRUD handler); every other grant can. A bot's effective computers (`grab-bot-computers`) are its direct grants plus every computer assigned to a non-archived workspace it's in (the coordinator's are all of the user's computers), the one it's on first: `bots.computer_id` (not a foreign key; null or a lost grant means its main one), so `computers[0]` is where its tasks run by default. The user switches it from the chat header (`(partials)/bot-computers-modal.tsx`, a ghost icon beside the model icon; the header names the current computer and the text model and provider under the bot's name) or by clicking a computer the bot named in a message: bots write `[computer:ID]` (`src/utils/computer-ref.ts`, turned into `#computer-ID` links by `link-computer-refs.ts`, rendered by `(partials)/computer-ref-chip.tsx`), both sharing `BotComputersContext` / `use-bot-computers`, saved by `api/admin/bots/set-computer.ts` (`setBotComputer`). Enums in `src/dict/computer-dict.ts`. +- **Opening a computer:** the app serves each computer to its owner under `/computers//` (`ComputerProxyPrefix`): `src/server.ts` hands those paths to `handleComputerProxy` (`src/functions/server/computers/`: `userAuth`, ownership via `grabUserComputer`, then the sandbox's address from `grabComputerMcpUrl`), which fetches plain requests (app session cookies dropped, `X-Forwarded-Host/Proto/Prefix` set so the sandbox builds its links and CDP address under the prefix, framing headers and redirects fixed) and bridges websockets (`computerProxySocket`, dispatched from `src/websocket/` when `ws.data.proxy` is set). `ComputerView` (`src/components/general/computer-view.tsx`) shows a computer in one of `ComputerViewModes` (browser `/browser-ui`, terminal `/terminal`, code `/code-server/`; more of the sandbox's apps can be added there), keeping visited modes loaded. In the chat, `(partials)/computer-viewer-modal.tsx` shows it, opened from a computer card's open button in the computers modal or a take-over button (`openComputer` on `BotComputersContext`). +- **Take-over:** a bot asks the user to take over a computer with `ask_user_to_take_over` (`src/harness/tools/computer/ask-to-take-over-tool.ts`, chat replies and tasks, defaulting to the task's computer). It writes `[take-over:ID:mode]` (`src/utils/take-over-ref.ts`; `withTakeOverRefs` adds it if the bot left it out and flags the message needs-you), linked by `link-computer-refs.ts` and rendered by `(partials)/take-over-chip.tsx`. The viewer then offers "Hand back to ", which sends a chat message so the bot carries on; a task stops and reports instead of waiting. - **Not built yet:** add-bot flow, routines, help, task steps (nothing writes `bot_task_steps` yet). ### Envs @@ -117,7 +119,7 @@ Envs are set in Settings, under Environment (super admin only), not in `.env`. T - **Automatic picks** (`rank-auto-models`): free models that can do the job (on the setting's provider when it has one; a provider without free models needs a picked model) (`model-serves-harness`: text needs tools), by vendor `AutoModelVendorPreference` (DeepSeek, Qwen, NVIDIA, Liquid), then other vendors that look competent (`AppData.HarnessAutoMin*`), then routers; within a vendor by intelligence index, then full-size before small, newest first. Transcription puts real speech-to-text models first. - **Calling:** `resolveHarnessCandidates` (pinned, then automatic) -> `withCandidates` tries them in order with process-wide cooldowns (`resolve/harness-cooldowns.ts`): rate limits and outages rest the model (until `Retry-After` when given), 401/402 rest the whole provider, 403/404 rest the model for an hour. Errors are `HarnessError` (`code` provider|unavailable|aborted, `retryable`), classified in `to-harness-error.ts`. Entry points: `text/generate-text.ts` (one turn), `text/run-text-loop.ts` (tool loop, sticks to one model, wraps up after `HarnessMaxToolSteps`, trims the oldest tool output before each turn to fit the model's context, `fit-transcript-to-context.ts`), `media/read-image.ts`, `media/read-media.ts` (documents locally via `fileToText`, images and audio through their harnesses), `audio/transcribe-audio.ts`, `audio/speak-text.ts`, `image/generate-image.ts`. A text model that can't take an image or audio part gets it read by the vision/transcription harness first (`prepare-messages-for-model.ts`). - **Per-bot models:** a worker bot inherits the defaults per harness unless it has a `bot_harness_settings` row (same fields as `harness_settings`, unique per bot and harness). `grabHarnessSetting({ harness, bot_id })` returns the bot's row (`source: "bot"`) or the default; the coordinator always gets the default. Every entry point takes `bot_id` and passes it on (text loop -> media reading/transcription), so a bot's run uses its own models throughout. Saved by `saveBotHarnessSetting` (`inherit` deletes the row); the page data is `grabBotHarnessOverview`. A bot's chat header has a ghost model icon (`(partials)/header-icon-button.tsx`; the text model and provider show under the bot's name) that opens a modal with every harness (`(partials)/bot-models-modal.tsx`); the modal and `/admin/bots/[bot_id]/models` both use `(partials)/bot-harness-form.tsx`. On the coordinator the modal edits the defaults themselves (`api/admin/harness/save-settings`, the same settings as Settings, under Models, without the providers): super admins only, the icon is disabled for everyone else. Every one of these wraps `settings/models/(partials)/harness-settings-form.tsx`: per harness a provider select first (`harness-provider-select.tsx`: any provider, one provider, or on a bot "Same as the default"), then the model select narrowed to it. Removing a provider (`deleteHarnessProvider`) puts default and bot settings that picked it or its models back on any provider (`save` prop; "Same as the default" shows when the overview item has `default`). -- **Tools:** `HarnessTool` (built per run with its bot bound): own tools validate with a zod `schema`; tools described elsewhere pass a JSON schema as `parameters` and validate themselves. `run-tool-call.ts` reports bad calls and failures back to the model as `Error: ...`. Bot tools in `src/harness/tools/bot/`: `update_memory`, `update_instructions` (`grabBotTools`, every run), `start_task` and, for the coordinator with workers, `delegate_task` (`grabBotChatTools`, chat replies only; the job's `requested_by_bot_id` is the coordinator, and `relayJobToRequester` posts the worker's result or failure in its conversation, pushed as `relay` by `pushBotNotice`), plus for the coordinator `assign_computer` (grant or revoke a worker's direct access, `add/remove-bot-to/from-computer`; access through a workspace stays and the result says so). `search_computers` (`src/harness/tools/computer/search-computers-tool.ts`, chat replies and tasks) runs one read-only command on all of a bot's computers at once over the sandbox REST API (`run-computer-command.ts`), each output under its `[computer:ID]`, secrets redacted, capped by `AppData.ComputerSearch*`. +- **Tools:** `HarnessTool` (built per run with its bot bound): own tools validate with a zod `schema`; tools described elsewhere pass a JSON schema as `parameters` and validate themselves. `run-tool-call.ts` reports bad calls and failures back to the model as `Error: ...`. Bot tools in `src/harness/tools/bot/`: `update_memory`, `update_instructions` (`grabBotTools`, every run), `start_task` and, for the coordinator with workers, `delegate_task` (`grabBotChatTools`, chat replies only; the job's `requested_by_bot_id` is the coordinator, and `relayJobToRequester` posts the worker's result or failure in its conversation, pushed as `relay` by `pushBotNotice`), `ask_user_to_take_over` (see Take-over), plus for the coordinator `assign_computer` (grant or revoke a worker's direct access, `add/remove-bot-to/from-computer`; access through a workspace stays and the result says so). `search_computers` (`src/harness/tools/computer/search-computers-tool.ts`, chat replies and tasks) runs one read-only command on all of a bot's computers at once over the sandbox REST API (`run-computer-command.ts`), each output under its `[computer:ID]`, secrets redacted, capped by `AppData.ComputerSearch*`. - **MCP:** `src/harness/mcp/`: `connectMcpServer` (Streamable HTTP, SSE fallback), `grabMcpTools` (each server tool becomes a `HarnessTool` with its JSON schema), `mcpResultToText` (text as is, images read by the vision harness, `isError` thrown). Computers are reached this way: `openComputerTools` (`src/harness/tools/computer/`) connects to the computer's AIO sandbox at `grabComputerMcpUrl` (container `metadata.mcp_url`, else the `MAIN_COMPUTER_URL` app setting for the main computer, for a dev server outside docker, e.g. `http://localhost:8170`, else `http://:/mcp`), leaves out `ComputerMcpExcludedTools`, and sets the computer's envs (see Envs). - **Replies and tasks:** `run-bot-reply.ts` answers from the last `BotReplyHistoryMessages` messages (`botMessagesToHarness`) with `ChatReplyNote`, the bot's computers (`compose-computers-note.ts`: what they are and that they're reached through `start_task`) and the chat tools, at most `BotReplyMaxToolSteps` steps, and finishes silently if the bot already had the last word (`enqueueBotReply` reuses a pending reply, so a burst of messages gets one). `run-bot-task.ts` runs the brief (`compose-job-brief.ts`) with `TaskRunNote`, the bot tools and the job's computer tools, closing the MCP connection after. Retryable harness errors go back to the queue's backoff, the rest are `BotJobFatalError`. - **API:** `api/admin/harness/` (`save-provider` also syncs, `sync-provider`, `delete-provider`, `save-settings`), super admin only, each returning the fresh `grabHarnessOverview()` page data. @@ -171,7 +173,7 @@ Server functions use `index.server.ts` alongside `index.tsx` (e.g. `single-docum | Directory | Description | | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | | `@moduletrace/twui` | **The** design system - external package. Contains layout primitives, elements, editors (TinyMCE), form components, hooks, SVGs, and utilities. Import from `@moduletrace/twui` or a deep path such as `@moduletrace/twui/layout/Button` | -| `src/components/general/` | Shared page components built on twui: `logo`, `logo-icon`, `wordmark`, `bot-stone` (+ `bot-stone-face`, `stone-eye-*`), `inner-page-hero-section`, `main-hero-section`, `admin-hero`, `login-shell`, `tagline-section`, media/image helpers | +| `src/components/general/` | Shared page components built on twui: `logo`, `logo-icon`, `wordmark`, `bot-stone` (+ `bot-stone-face`, `stone-eye-*`), `user-avatar` (round photo or initials, `sm|md|lg`), `computer-view` (+ `computer-view-frame`, `computer-view-mode-button`), `inner-page-hero-section`, `main-hero-section`, `admin-hero`, `login-shell`, `tagline-section`, media/image helpers | ## UI & Styling Rules (MANDATORY) diff --git a/src/bots-queue/describe-job-step.test.ts b/src/bots-queue/describe-job-step.test.ts index 4419614..addc1b0 100644 --- a/src/bots-queue/describe-job-step.test.ts +++ b/src/bots-queue/describe-job-step.test.ts @@ -5,11 +5,11 @@ import describeJobStep from "./describe-job-step"; const candidate = { model: { slug: "deepseek/deepseek-chat:free" } } as HarnessCandidate; describe("describeJobStep", () => { - test("names each tool once, never its arguments", () => { + test("says what each tool does once, never its arguments", () => { expect( describeJobStep({ index: 2, - text: "Let me look.", + text: null, candidate, tool_calls: [ { id: "a", name: "sandbox_execute_bash", arguments: '{"command":"cat secret"}' }, @@ -19,7 +19,7 @@ describe("describeJobStep", () => { }), ).toEqual({ kind: "tool", - text: "Using sandbox_execute_bash, update_memory", + text: "Running a command, Updating memory", metadata: { step: 2, model: "deepseek/deepseek-chat:free", @@ -28,6 +28,32 @@ describe("describeJobStep", () => { }); }); + test("prefers what the model said it's doing", () => { + expect( + describeJobStep({ + index: 0, + text: "Opening the Gmail tab.\nThen reading it.", + candidate, + tool_calls: [{ id: "a", name: "browser_navigate", arguments: "{}" }], + })?.text, + ).toBe("Opening the Gmail tab."); + }); + + test("labels tool families and falls back to the name", () => { + expect( + describeJobStep({ + index: 0, + text: "", + candidate, + tool_calls: [ + { id: "a", name: "browser_navigate", arguments: "{}" }, + { id: "b", name: "browser_click", arguments: "{}" }, + { id: "c", name: "some_mcp_tool", arguments: "{}" }, + ], + })?.text, + ).toBe("Using the browser, Using some_mcp_tool"); + }); + test("keeps the first line of a text-only turn, cut to one line", () => { const step = describeJobStep({ index: 0, diff --git a/src/bots-queue/describe-job-step.ts b/src/bots-queue/describe-job-step.ts index 2b3ae42..db67c69 100644 --- a/src/bots-queue/describe-job-step.ts +++ b/src/bots-queue/describe-job-step.ts @@ -1,15 +1,21 @@ -import type { BotJobUpdateKind } from "@/src/dict/bot-queue-dict"; +import { + JobStepToolLabels, + JobStepToolPrefixLabels, + type BotJobUpdateKind, +} from "@/src/dict/bot-queue-dict"; import type { TextLoopStep } from "@/src/harness/text/run-text-loop"; /** - * Longest step line kept: the panel shows one line per step. + * Longest step line kept: the chat shows one line per step. */ const MaxStepChars = 200; /** - * One model turn as a `bots_queue_updates` row: the tools it called, or - * the first line of what it wrote. Null for an empty turn. Never the tool - * arguments, which can hold anything the bot typed. + * One model turn as a `bots_queue_updates` row, read by the user while the + * bot works: the first line the model wrote (models often say what + * they're about to do), else what its tools do in plain words. Null for an + * empty turn. Never the tool arguments, which can hold anything the bot + * typed. */ export default function describeJobStep({ text, @@ -22,24 +28,34 @@ export default function describeJobStep({ metadata: Record; } | null { const metadata = { step: index, model: candidate.model.slug }; + const line = firstLine({ text }); if (tool_calls.length) { const tools = [...new Set(tool_calls.map((call) => call.name))]; + const labels = [...new Set(tools.map((name) => toolLabel({ name })))]; return { kind: "tool", - text: `Using ${tools.join(", ")}`, + text: line || labels.join(", "), metadata: { ...metadata, tools }, }; } + if (!line) return null; + + return { kind: "note", text: line, metadata }; +} + +function firstLine({ text }: { text: string | null }) { const line = text?.trim().split("\n").find((l) => l.trim())?.trim(); if (!line) return null; - return { - kind: "note", - text: line.length > MaxStepChars ? `${line.slice(0, MaxStepChars - 1)}…` : line, - metadata, - }; + return line.length > MaxStepChars ? `${line.slice(0, MaxStepChars - 1)}…` : line; +} + +function toolLabel({ name }: { name: string }) { + const prefix = Object.keys(JobStepToolPrefixLabels).find((p) => name.startsWith(p)); + + return JobStepToolLabels[name] || (prefix && JobStepToolPrefixLabels[prefix]) || `Using ${name}`; } diff --git a/src/bots-queue/run-bot-reply.ts b/src/bots-queue/run-bot-reply.ts index a18cfaf..cebc0a3 100644 --- a/src/bots-queue/run-bot-reply.ts +++ b/src/bots-queue/run-bot-reply.ts @@ -1,6 +1,6 @@ import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db"; import { AppData } from "@/src/data/app-data"; -import { ChatReplyNote, CoordinatorReplyNote } from "@/src/data/harness-prompts"; +import { ChatReplyNote, CoordinatorReplyNote, CoordinatorSoloNote } from "@/src/data/harness-prompts"; import grabBotMessages from "@/src/functions/backend/db/bots/grab-bot-messages"; import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers"; @@ -9,6 +9,7 @@ import grabBotChatTools from "@/src/harness/tools/bot/grab-bot-chat-tools"; import botMessagesToHarness from "./bot-messages-to-harness"; import composeComputersNote from "./compose-computers-note"; import loadAttachmentParts from "./load-attachment-parts"; +import withTakeOverRefs from "./with-take-over-refs"; import type { BotJobContext, BotJobOutcome } from "./types"; type Params = BotJobContext & { @@ -40,12 +41,32 @@ export default async function runBotReply({ const prompt = await grabBotPrompt({ bot_id: bot.id, user_id: bot.user_id }); const computers = await grabBotComputers({ bot_id: bot.id, user_id: bot.user_id }); + const take_overs: string[] = []; + + const tools = await grabBotChatTools({ + bot, + user_id: bot.user_id, + computers, + message_id: job.message_id || undefined, + onTakeOver: ({ ref }) => take_overs.push(ref), + }); + + /** + * delegate_task is only there while the coordinator has workers. + */ + const crew_note = + bot.kind != "coordinator" + ? "" + : tools.some((tool) => tool.name == "delegate_task") + ? CoordinatorReplyNote + : CoordinatorSoloNote; + const result = await runTextLoop({ system: [ prompt, ChatReplyNote, composeComputersNote({ computers, is_coordinator: bot.kind == "coordinator" }), - bot.kind == "coordinator" ? CoordinatorReplyNote : "", + crew_note, `Now: ${new Date().toUTCString()}.`, ] .filter(Boolean) @@ -54,17 +75,12 @@ export default async function runBotReply({ messages: history, parts: await loadAttachmentParts({ messages: history }), }), - tools: await grabBotChatTools({ - bot, - user_id: bot.user_id, - computers, - message_id: job.message_id || undefined, - }), + tools, max_steps: AppData.BotReplyMaxToolSteps, bot_id: bot.id, signal, onStep: report, }); - return { result: result.text }; + return withTakeOverRefs({ text: result.text, refs: take_overs }); } diff --git a/src/bots-queue/run-bot-task.ts b/src/bots-queue/run-bot-task.ts index 0045f85..81fd9d6 100644 --- a/src/bots-queue/run-bot-task.ts +++ b/src/bots-queue/run-bot-task.ts @@ -4,9 +4,11 @@ import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt"; import runTextLoop from "@/src/harness/text/run-text-loop"; import openComputerTools from "@/src/harness/tools/computer/open-computer-tools"; import grabBotTools from "@/src/harness/tools/bot/grab-bot-tools"; +import askToTakeOverTool from "@/src/harness/tools/computer/ask-to-take-over-tool"; import searchComputersTool from "@/src/harness/tools/computer/search-computers-tool"; import computerRef from "@/src/utils/computer-ref"; import composeJobBrief from "./compose-job-brief"; +import withTakeOverRefs from "./with-take-over-refs"; import type { BotJobContext, BotJobOutcome } from "./types"; type Params = BotJobContext & { @@ -17,6 +19,8 @@ type Params = BotJobContext & { * Works through a task: the bot follows its brief with its full prompt, * its own tools, search_computers across all its computers and, when the * job has a computer, that computer's tools (over MCP), until it answers. That answer is the report for the user. + * When only the user can go on (a login, a captcha), the bot asks them to + * take over (`ask_user_to_take_over`) and the report needs them. * Every model turn is reported as a step (`report`). */ export default async function runBotTask({ @@ -33,13 +37,22 @@ export default async function runBotTask({ try { const prompt = await grabBotPrompt({ bot_id: bot.id, user_id: bot.user_id }); const computers = await grabBotComputers({ bot_id: bot.id, user_id: bot.user_id }); + const take_overs: string[] = []; + + /** + * A take-over defaults to the computer the task runs on. + */ + const take_over_computers = [ + ...computers.filter((c) => c.id == job.computer_id), + ...computers.filter((c) => c.id != job.computer_id), + ]; const result = await runTextLoop({ system: [ prompt, TaskRunNote, computer - ? `You're working on the computer "${computer.name}" (a Linux sandbox with a shell, files, code runners and a browser). Use its tools; don't ask the user to run anything.` + ? `You're working on the computer "${computer.name}" (a Linux sandbox with a shell, files, code runners and a browser). Use its tools; don't ask the user to run anything. When only the user can go on (a login, a captcha, a code sent to them), ask them to take over with ask_user_to_take_over.` : `You have no computer for this task.`, computers.length > 1 ? `search_computers runs a read-only command on every computer you have. When you name one in the report, write its ref, e.g. ${computerRef({ id: computers[0]!.id || 0 })}: the user can click it to switch you to that computer.` @@ -64,7 +77,16 @@ export default async function runBotTask({ ], tools: [ ...grabBotTools({ user_id: bot.user_id, bot_id: bot.id }), - ...(computers.length ? [searchComputersTool({ bot_id: bot.id, computers })] : []), + ...(computers.length + ? [ + searchComputersTool({ bot_id: bot.id, computers }), + askToTakeOverTool({ + computers: take_over_computers, + in_task: true, + onAsk: ({ ref }) => take_overs.push(ref), + }), + ] + : []), ...(computer?.tools || []), ], bot_id: bot.id, @@ -73,7 +95,10 @@ export default async function runBotTask({ report({ ...step, text: step.text && computer ? computer.redact(step.text) : step.text }), }); - return { result: computer ? computer.redact(result.text) : result.text }; + return withTakeOverRefs({ + text: computer ? computer.redact(result.text) : result.text, + refs: take_overs, + }); } finally { await computer?.close(); } diff --git a/src/bots-queue/with-take-over-refs.test.ts b/src/bots-queue/with-take-over-refs.test.ts new file mode 100644 index 0000000..1cbf034 --- /dev/null +++ b/src/bots-queue/with-take-over-refs.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from "bun:test"; +import takeOverRef from "@/src/utils/take-over-ref"; +import withTakeOverRefs from "./with-take-over-refs"; + +const ref = takeOverRef({ id: 1, mode: "browser" }); + +describe("withTakeOverRefs", () => { + test("adds a take-over the bot didn't write, once", () => { + expect(withTakeOverRefs({ text: "Please log in to Gmail.", refs: [ref, ref] })).toEqual({ + result: `Please log in to Gmail.\n\n${ref}`, + needs_you: true, + }); + }); + + test("keeps one the bot wrote itself", () => { + const text = `Log in here: ${ref}`; + + expect(withTakeOverRefs({ text, refs: [ref] })).toEqual({ result: text, needs_you: true }); + }); + + test("leaves a message without a take-over alone", () => { + expect(withTakeOverRefs({ text: "Done.", refs: [] })).toEqual({ result: "Done.", needs_you: false }); + }); +}); diff --git a/src/bots-queue/with-take-over-refs.ts b/src/bots-queue/with-take-over-refs.ts new file mode 100644 index 0000000..a62c8d2 --- /dev/null +++ b/src/bots-queue/with-take-over-refs.ts @@ -0,0 +1,21 @@ +type Params = { + text: string; + /** + * The take-over refs the bot asked for in this run (`askToTakeOverTool`). + */ + refs: string[]; +}; + +/** + * A job's message with every take-over the bot asked for in it: refs the + * bot didn't write itself are added at the end, so the user always gets + * the button. A message asking for a take-over needs the user. + */ +export default function withTakeOverRefs({ text, refs }: Params) { + const missing = [...new Set(refs)].filter((ref) => !text.includes(ref)); + + return { + result: [text.trim(), ...missing].filter(Boolean).join("\n\n"), + needs_you: refs.length > 0, + }; +} diff --git a/src/components/general/computer-view-frame.tsx b/src/components/general/computer-view-frame.tsx new file mode 100644 index 0000000..81fc6b7 --- /dev/null +++ b/src/components/general/computer-view-frame.tsx @@ -0,0 +1,29 @@ +import { ComputerViewModes, type ComputerViewMode } from "@/src/dict/computer-dict"; +import computerViewUrl from "@/src/utils/computer-view-url"; + +type Props = { + computer_id: number; + name: string; + mode: ComputerViewMode; + /** + * The mode on show; the others stay loaded but hidden. + */ + active: boolean; +}; + +/** + * One of a computer's apps in a frame. A raw iframe: twui has no + * equivalent. + */ +export default function ComputerViewFrame({ computer_id, name, mode, active }: Props) { + const title = ComputerViewModes.find((m) => m.value == mode)?.title || mode; + + return ( +