Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 10 Wiki Activity

Open computers from the chat, let bots ask for a take-over, show task progress

Browse Source 
- Serve each computer to its owner under /computers/<id>/ (HTTP and
  websockets proxied to the AIO sandbox) and show it in a shared
  ComputerView with browser, terminal and code modes, opened from the
  computers modal or a take-over button in the thread.
- ask_user_to_take_over lets a bot hand a computer to the user for a
  login or captcha; the message gets a take-over button and needs-you,
  and the viewer hands it back with a chat message.
- While a task runs the thread shows "<bot> is working..." with the
  task's latest step; steps read as plain words, not tool names.
- A coordinator with no workers no longer claims to delegate, and bots
  describe started tasks in the first person.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-19 05:35:44 +01:00
co-authored by Claude Opus 5.5
parent b1023ba8ff
commit 56daf0c462
44 changed files with 1289 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bots-queue/describe-job-step.test.ts
+29 -3
View File
@@ -5,11 +5,11 @@ import describeJobStep from "./describe-job-step";
const candidate = { model: { slug: "deepseek/deepseek-chat:free" } } as HarnessCandidate;
describe("describeJobStep", () => {
test("names each tool once, never its arguments", () => {
test("says what each tool does once, never its arguments", () => {
expect(
describeJobStep({
index: 2,
text: "Let me look.",
text: null,
candidate,
tool_calls: [
{ id: "a", name: "sandbox_execute_bash", arguments: '{"command":"cat secret"}' },
@@ -19,7 +19,7 @@ describe("describeJobStep", () => {
}),
).toEqual({
kind: "tool",
text: "Using sandbox_execute_bash, update_memory",
text: "Running a command, Updating memory",
metadata: {
step: 2,
model: "deepseek/deepseek-chat:free",
@@ -28,6 +28,32 @@ describe("describeJobStep", () => {
});
});
test("prefers what the model said it's doing", () => {
expect(
describeJobStep({
index: 0,
text: "Opening the Gmail tab.\nThen reading it.",
candidate,
tool_calls: [{ id: "a", name: "browser_navigate", arguments: "{}" }],
})?.text,
).toBe("Opening the Gmail tab.");
});
test("labels tool families and falls back to the name", () => {
expect(
describeJobStep({
index: 0,
text: "",
candidate,
tool_calls: [
{ id: "a", name: "browser_navigate", arguments: "{}" },
{ id: "b", name: "browser_click", arguments: "{}" },
{ id: "c", name: "some_mcp_tool", arguments: "{}" },
],
})?.text,
).toBe("Using the browser, Using some_mcp_tool");
});
test("keeps the first line of a text-only turn, cut to one line", () => {
const step = describeJobStep({
index: 0,
src/bots-queue/describe-job-step.ts
+27 -11
View File
@@ -1,15 +1,21 @@
import type { BotJobUpdateKind } from "@/src/dict/bot-queue-dict";
import {
JobStepToolLabels,
JobStepToolPrefixLabels,
type BotJobUpdateKind,
} from "@/src/dict/bot-queue-dict";
import type { TextLoopStep } from "@/src/harness/text/run-text-loop";
/**
* Longest step line kept: the panel shows one line per step.
* Longest step line kept: the chat shows one line per step.
*/
const MaxStepChars = 200;
/**
* One model turn as a `bots_queue_updates` row: the tools it called, or
* the first line of what it wrote. Null for an empty turn. Never the tool
* arguments, which can hold anything the bot typed.
* One model turn as a `bots_queue_updates` row, read by the user while the
* bot works: the first line the model wrote (models often say what
* they're about to do), else what its tools do in plain words. Null for an
* empty turn. Never the tool arguments, which can hold anything the bot
* typed.
*/
export default function describeJobStep({
text,
@@ -22,24 +28,34 @@ export default function describeJobStep({
metadata: Record<string, unknown>;
} | null {
const metadata = { step: index, model: candidate.model.slug };
const line = firstLine({ text });
if (tool_calls.length) {
const tools = [...new Set(tool_calls.map((call) => call.name))];
const labels = [...new Set(tools.map((name) => toolLabel({ name })))];
return {
kind: "tool",
text: `Using ${tools.join(", ")}`,
text: line || labels.join(", "),
metadata: { ...metadata, tools },
};
}
if (!line) return null;
return { kind: "note", text: line, metadata };
}
function firstLine({ text }: { text: string | null }) {
const line = text?.trim().split("\n").find((l) => l.trim())?.trim();
if (!line) return null;
return {
kind: "note",
text: line.length > MaxStepChars ? `${line.slice(0, MaxStepChars - 1)}…` : line,
metadata,
};
return line.length > MaxStepChars ? `${line.slice(0, MaxStepChars - 1)}…` : line;
}
function toolLabel({ name }: { name: string }) {
const prefix = Object.keys(JobStepToolPrefixLabels).find((p) => name.startsWith(p));
return JobStepToolLabels[name] || (prefix && JobStepToolPrefixLabels[prefix]) || `Using ${name}`;
}
src/bots-queue/run-bot-reply.ts
+25 -9
View File
@@ -1,6 +1,6 @@
import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
import { AppData } from "@/src/data/app-data";
import { ChatReplyNote, CoordinatorReplyNote } from "@/src/data/harness-prompts";
import { ChatReplyNote, CoordinatorReplyNote, CoordinatorSoloNote } from "@/src/data/harness-prompts";
import grabBotMessages from "@/src/functions/backend/db/bots/grab-bot-messages";
import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt";
import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers";
@@ -9,6 +9,7 @@ import grabBotChatTools from "@/src/harness/tools/bot/grab-bot-chat-tools";
import botMessagesToHarness from "./bot-messages-to-harness";
import composeComputersNote from "./compose-computers-note";
import loadAttachmentParts from "./load-attachment-parts";
import withTakeOverRefs from "./with-take-over-refs";
import type { BotJobContext, BotJobOutcome } from "./types";
type Params = BotJobContext & {
@@ -40,12 +41,32 @@ export default async function runBotReply({
const prompt = await grabBotPrompt({ bot_id: bot.id, user_id: bot.user_id });
const computers = await grabBotComputers({ bot_id: bot.id, user_id: bot.user_id });
const take_overs: string[] = [];
const tools = await grabBotChatTools({
bot,
user_id: bot.user_id,
computers,
message_id: job.message_id || undefined,
onTakeOver: ({ ref }) => take_overs.push(ref),
});
/**
* delegate_task is only there while the coordinator has workers.
*/
const crew_note =
bot.kind != "coordinator"
? ""
: tools.some((tool) => tool.name == "delegate_task")
? CoordinatorReplyNote
: CoordinatorSoloNote;
const result = await runTextLoop({
system: [
prompt,
ChatReplyNote,
composeComputersNote({ computers, is_coordinator: bot.kind == "coordinator" }),
bot.kind == "coordinator" ? CoordinatorReplyNote : "",
crew_note,
`Now: ${new Date().toUTCString()}.`,
]
.filter(Boolean)
@@ -54,17 +75,12 @@ export default async function runBotReply({
messages: history,
parts: await loadAttachmentParts({ messages: history }),
}),
tools: await grabBotChatTools({
bot,
user_id: bot.user_id,
computers,
message_id: job.message_id || undefined,
}),
tools,
max_steps: AppData.BotReplyMaxToolSteps,
bot_id: bot.id,
signal,
onStep: report,
});
return { result: result.text };
return withTakeOverRefs({ text: result.text, refs: take_overs });
}
src/bots-queue/run-bot-task.ts
+28 -3
View File
@@ -4,9 +4,11 @@ import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt";
import runTextLoop from "@/src/harness/text/run-text-loop";
import openComputerTools from "@/src/harness/tools/computer/open-computer-tools";
import grabBotTools from "@/src/harness/tools/bot/grab-bot-tools";
import askToTakeOverTool from "@/src/harness/tools/computer/ask-to-take-over-tool";
import searchComputersTool from "@/src/harness/tools/computer/search-computers-tool";
import computerRef from "@/src/utils/computer-ref";
import composeJobBrief from "./compose-job-brief";
import withTakeOverRefs from "./with-take-over-refs";
import type { BotJobContext, BotJobOutcome } from "./types";
type Params = BotJobContext & {
@@ -17,6 +19,8 @@ type Params = BotJobContext & {
* Works through a task: the bot follows its brief with its full prompt,
* its own tools, search_computers across all its computers and, when the
* job has a computer, that computer's tools (over MCP), until it answers. That answer is the report for the user.
* When only the user can go on (a login, a captcha), the bot asks them to
* take over (`ask_user_to_take_over`) and the report needs them.
* Every model turn is reported as a step (`report`).
*/
export default async function runBotTask({
@@ -33,13 +37,22 @@ export default async function runBotTask({
try {
const prompt = await grabBotPrompt({ bot_id: bot.id, user_id: bot.user_id });
const computers = await grabBotComputers({ bot_id: bot.id, user_id: bot.user_id });
const take_overs: string[] = [];
/**
* A take-over defaults to the computer the task runs on.
*/
const take_over_computers = [
...computers.filter((c) => c.id == job.computer_id),
...computers.filter((c) => c.id != job.computer_id),
];
const result = await runTextLoop({
system: [
prompt,
TaskRunNote,
computer
? `You're working on the computer "${computer.name}" (a Linux sandbox with a shell, files, code runners and a browser). Use its tools; don't ask the user to run anything.`
? `You're working on the computer "${computer.name}" (a Linux sandbox with a shell, files, code runners and a browser). Use its tools; don't ask the user to run anything. When only the user can go on (a login, a captcha, a code sent to them), ask them to take over with ask_user_to_take_over.`
: `You have no computer for this task.`,
computers.length > 1
? `search_computers runs a read-only command on every computer you have. When you name one in the report, write its ref, e.g. ${computerRef({ id: computers[0]!.id || 0 })}: the user can click it to switch you to that computer.`
@@ -64,7 +77,16 @@ export default async function runBotTask({
],
tools: [
...grabBotTools({ user_id: bot.user_id, bot_id: bot.id }),
...(computers.length ? [searchComputersTool({ bot_id: bot.id, computers })] : []),
...(computers.length
? [
searchComputersTool({ bot_id: bot.id, computers }),
askToTakeOverTool({
computers: take_over_computers,
in_task: true,
onAsk: ({ ref }) => take_overs.push(ref),
}),
]
: []),
...(computer?.tools || []),
],
bot_id: bot.id,
@@ -73,7 +95,10 @@ export default async function runBotTask({
report({ ...step, text: step.text && computer ? computer.redact(step.text) : step.text }),
});
return { result: computer ? computer.redact(result.text) : result.text };
return withTakeOverRefs({
text: computer ? computer.redact(result.text) : result.text,
refs: take_overs,
});
} finally {
await computer?.close();
}
src/bots-queue/with-take-over-refs.test.ts
+24
View File
@@ -0,0 +1,24 @@
import { describe, expect, test } from "bun:test";
import takeOverRef from "@/src/utils/take-over-ref";
import withTakeOverRefs from "./with-take-over-refs";
const ref = takeOverRef({ id: 1, mode: "browser" });
describe("withTakeOverRefs", () => {
test("adds a take-over the bot didn't write, once", () => {
expect(withTakeOverRefs({ text: "Please log in to Gmail.", refs: [ref, ref] })).toEqual({
result: `Please log in to Gmail.\n\n${ref}`,
needs_you: true,
});
});
test("keeps one the bot wrote itself", () => {
const text = `Log in here: ${ref}`;
expect(withTakeOverRefs({ text, refs: [ref] })).toEqual({ result: text, needs_you: true });
});
test("leaves a message without a take-over alone", () => {
expect(withTakeOverRefs({ text: "Done.", refs: [] })).toEqual({ result: "Done.", needs_you: false });
});
});
src/bots-queue/with-take-over-refs.ts
+21
View File
@@ -0,0 +1,21 @@
type Params = {
text: string;
/**
* The take-over refs the bot asked for in this run (`askToTakeOverTool`).
*/
refs: string[];
};
/**
* A job's message with every take-over the bot asked for in it: refs the
* bot didn't write itself are added at the end, so the user always gets
* the button. A message asking for a take-over needs the user.
*/
export default function withTakeOverRefs({ text, refs }: Params) {
const missing = [...new Set(refs)].filter((ref) => !text.includes(ref));
return {
result: [text.trim(), ...missing].filter(Boolean).join("\n\n"),
needs_you: refs.length > 0,
};
}