Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 32 Wiki Activity

Keep a record of each run's tool calls so bots can't claim work they never did

Browse Source 
The wrap-up turn now gets the run's actual tool calls, each bot message
keeps them as bot_messages.tool_log, and history follows recorded bot
messages with an [App record] line. The coordinator also gets update_bot
in workspaces, so it can fill a bot's core prompt from there.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-25 11:32:56 +01:00
co-authored by Claude Opus 5.5
parent 93896646c7
commit d7f88d8dd3
22 changed files with 293 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bots-queue/bot-messages-to-harness.test.ts
+45
View File
@@ -152,4 +152,49 @@ describe("botMessagesToHarness with mentions", () => {
{ role: "assistant", content: "[bot:3] is on it" },
]);
});
test("follows a recorded bot message with the app's record of its tools", () => {
expect(
botMessagesToHarness({
messages: [
message("user", "fill in its prompt"),
{
...message("bot", "Done."),
tool_log: JSON.stringify([
{ name: "fetch_url", ok: true },
{ name: "fetch_url", ok: false },
]),
},
message("user", "and the other one?"),
{ ...message("bot", "Saved too."), tool_log: "[]" },
],
}),
).toEqual([
{ role: "user", content: "fill in its prompt" },
{
role: "assistant",
content: "Done.\n\n[App record: this message's run called fetch_url x2 (1 failed).]",
},
{ role: "user", content: "and the other one?" },
{
role: "assistant",
content: "Saved too.\n\n[App record: this message's run called no tools.]",
},
]);
});
test("adds no record for messages whose run wasn't recorded", () => {
expect(
botMessagesToHarness({
messages: [
message("user", "hi"),
{ ...message("bot", "hello"), tool_log: undefined },
{ ...message("bot", "again"), tool_log: "not json" },
],
}),
).toEqual([
{ role: "user", content: "hi" },
{ role: "assistant", content: "hello\n\nagain" },
]);
});
});
src/bots-queue/bot-messages-to-harness.ts
+37 -4
View File
@@ -1,12 +1,14 @@
import type { HarnessContentPart, HarnessMessage } from "@/src/harness/types";
import describeToolUses from "@/src/harness/text/describe-tool-uses";
import type { HarnessContentPart, HarnessMessage, HarnessToolUse } from "@/src/harness/types";
import type { BotMessageWithAttachments } from "@/src/types";
import describeMessageRefs, { type MessageRefNames } from "@/src/utils/describe-message-refs";
import { AppRecordPrefix } from "@/src/utils/strip-app-records";
import describeAttachment from "./describe-attachment";
type Params = {
messages: Pick<
BotMessageWithAttachments,
"id" | "author" | "text" | "attachments" | "reply_to"
"id" | "author" | "text" | "attachments" | "reply_to" | "tool_log"
>[];
/**
* Loaded files by message id (`loadAttachmentParts`). Files of other
@@ -27,7 +29,10 @@ type Turn = { role: "user" | "assistant"; parts: HarnessContentPart[] };
* (with their files) as user turns, the bot's as its own. Back-to-back
* messages from the same side are joined, since some providers reject
* repeated roles. Leading bot messages are dropped so the conversation
* starts with the user. A turn that is only text stays a string.
* starts with the user. A turn that is only text stays a string. A bot
* message whose run was recorded (`tool_log`) is followed by the app's
* record of the tools it called, so the bot can tell what its past
* messages actually did from what they say.
*/
export default function botMessagesToHarness({ messages, parts, refs }: Params): HarnessMessage[] {
const turns: Turn[] = [];
@@ -78,7 +83,13 @@ function messageParts({
const text = replyContext({ message, text: mentions ? `${own}\n\n${mentions}` : own });
const content: HarnessContentPart[] = text ? [{ type: "text", text }] : [];
if (role == "assistant" || !message.attachments?.length) return content;
if (role == "assistant") {
const record = text ? appRecord({ tool_log: message.tool_log }) : "";
return record ? [...content, { type: "text", text: record }] : content;
}
if (!message.attachments?.length) return content;
const loaded = parts?.get(Number(message.id));
@@ -93,6 +104,28 @@ function messageParts({
];
}
/**
* The app's line after a bot message naming the tools its run called, or
* nothing when the run wasn't recorded.
*/
function appRecord({ tool_log }: { tool_log?: string | null }) {
if (!tool_log) return "";
let uses: HarnessToolUse[];
try {
uses = JSON.parse(tool_log);
} catch {
return "";
}
if (!Array.isArray(uses)) return "";
const called = uses.length ? describeToolUses({ uses }) : "no tools";
return `${AppRecordPrefix} this message's run called ${called}.]`;
}
/**
* Names the message a user turn answers, so the bot reads the reply in
* context even when the quoted message is further back. A reply with
src/bots-queue/process-bot-job.ts
+1
View File
@@ -212,6 +212,7 @@ export default async function processBotJob({
needs_you: outcome.needs_you,
silent: outcome.silent,
attachments: outcome.attachments,
tool_log: outcome.tool_log,
});
await pushBotNotice({ bot_id, ...completion });
src/bots-queue/run-bot-reply.ts
+6 -1
View File
@@ -33,6 +33,7 @@ import jobSessionId from "@/src/functions/backend/db/bots-queue/job-session-id";
import loadAttachmentParts from "./load-attachment-parts";
import grabMessageRefNames from "@/src/functions/backend/db/bots/grab-message-ref-names";
import withTakeOverRefs from "./with-take-over-refs";
import stripAppRecords from "@/src/utils/strip-app-records";
import hasUnansweredMessage from "./has-unanswered-message";
import grabAnsweredMessageId from "@/src/functions/backend/db/bots-queue/grab-answered-message-id";
import markReplyAnswered from "@/src/functions/backend/db/bots-queue/mark-reply-answered";
@@ -269,8 +270,12 @@ export default async function runBotReply({
}
return {
...withTakeOverRefs({ text: result.text, refs: [...take_overs, ...proposals] }),
...withTakeOverRefs({
text: stripAppRecords({ text: result.text }),
refs: [...take_overs, ...proposals],
}),
attachments: attachments.length ? attachments : undefined,
tool_log: result.tool_uses,
};
} finally {
await extensions.close();
src/bots-queue/run-bot-task.ts
+5 -1
View File
@@ -34,6 +34,7 @@ import composeJobBrief from "./compose-job-brief";
import grabSessionCarryOver from "./grab-session-carry-over";
import jobSessionId from "@/src/functions/backend/db/bots-queue/job-session-id";
import withTakeOverRefs from "./with-take-over-refs";
import stripAppRecords from "@/src/utils/strip-app-records";
import ScheduledRunFailedError from "./scheduled-run-failed-error";
import type { BotJobContext, BotJobOutcome } from "./types";
import computerRunDirs from "@/src/utils/computer-work-dirs";
@@ -272,12 +273,15 @@ export default async function runBotTask({
throw new ScheduledRunFailedError(computer ? computer.redact(failure) : failure);
}
const text = stripAppRecords({ text: result.text });
return {
...withTakeOverRefs({
text: computer ? computer.redact(result.text) : result.text,
text: computer ? computer.redact(text) : text,
refs: take_overs,
}),
attachments: attachments.length ? attachments : undefined,
tool_log: result.tool_uses,
};
} finally {
await extensions?.close();
src/bots-queue/types.ts
+6
View File
@@ -1,5 +1,6 @@
import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
import type { TextLoopStep } from "@/src/harness/text/run-text-loop";
import type { HarnessToolUse } from "@/src/harness/types";
export type BotJobContext = {
job: BUN_SQLITE_COBBLE_BOTS_QUEUE;
@@ -45,6 +46,11 @@ export type BotJobOutcome = {
* ids), attached to the result message.
*/
attachments?: number[];
/**
* Every tool call the run made (`runTextLoop`'s `tool_uses`), kept on
* the result message as `tool_log`.
*/
tool_log?: HarnessToolUse[];
};
export type BotJobRunner = (context: BotJobContext) => Promise<BotJobOutcome>;