Moduletrace/cobble
2
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases 11 Wiki Activity

Make routines act on their own and check every bot action

Browse Source 
Scheduled runs are pre-approved: they carry out their brief instead of
drafting for confirmation. A run that can't do its brief calls
report_run_failed and is retried 5 and 10 minutes later; only then does
the user get one "failed after N tries" message, and the routine keeps
why in last_error.

Every reply and task now gets ActionCheckNote: look for an existing
record before creating one, and verify an action took effect before
saying it's done. Memory and instructions saves are read back, and
create_bot, create_schedule, start_task and delegate_task refuse
duplicates.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
tben
2026-10-20 08:45:49 +01:00
co-authored by Claude Opus 5.5
parent 6c57fae444
commit 535adb6e16
25 changed files with 448 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bots-queue/process-bot-job.ts
+28 -3
View File
@@ -9,6 +9,8 @@ import { logError } from "@/src/functions/backend/logger";
import createKeyLock from "@/src/utils/create-key-lock";
import HarnessError from "@/src/harness/harness-error";
import deferBotJob from "@/src/functions/backend/db/bots-queue/defer-bot-job";
import recordCronJobFailure from "@/src/functions/backend/db/cron-jobs/record-cron-job-failure";
import scheduledRunRetryAt from "@/src/utils/scheduled-run-retry-at";
import {
BotComputerBusyStatusLine,
BotReplyJobTitle,
@@ -56,7 +58,9 @@ type Params = {
* Runs one claimed job to the end: keeps its lock alive, waits for its
* computer, runs it, then completes or fails it and pushes the outcome to
* the user. If the job is cancelled or lost mid-way, its result is
* dropped.
* dropped. A failed scheduled run (a routine) is retried with longer gaps
* (`scheduledRunRetryAt`); once out of tries the user is told, and the
* schedule keeps why (`recordCronJobFailure`).
*/
export default async function processBotJob({
job,
@@ -67,6 +71,8 @@ export default async function processBotJob({
const job_id = job.id;
const bot_id = job.bot_id;
const params = parseJobJson({ text: job.params }) || {};
const schedule_id = Number(params.schedule_id) || null;
/**
* Its face changes only for the user it works for.
@@ -133,7 +139,7 @@ export default async function processBotJob({
return await runner({
job,
params: parseJobJson({ text: job.params }) || {},
params,
checkpoint: parseJobJson({ text: job.checkpoint }),
signal: controller.signal,
update,
@@ -223,11 +229,17 @@ export default async function processBotJob({
return;
}
const message = error instanceof Error ? error.message : String(error);
const tries = Number(job.attempts) || 1;
const failed = await failBotJob({
job_id,
worker,
error: error instanceof Error ? error.message : String(error),
error: message,
fatal: error instanceof BotJobFatalError,
failure_message: schedule_id
? `My scheduled run "${job.title || "Scheduled task"}" failed after ${tries} ${tries == 1 ? "try" : "tries"}: ${message}`
: undefined,
}).catch(async (fail_error) => {
await logError({
title: `Bot job #${job_id} couldn't be failed`,
@@ -240,6 +252,19 @@ export default async function processBotJob({
await pushBotNotice({ bot_id, ...failed.notice });
}
if (schedule_id && failed?.status == "failed") {
await recordCronJobFailure({ cron_job_id: schedule_id, job_id, error: message }).catch(
() => {},
);
}
/**
* A routine's retry waits long enough for an outage to pass.
*/
if (schedule_id && failed?.status == "pending") {
await deferBotJob({ job_id, worker, until: scheduledRunRetryAt({ attempts: failed.attempts }) });
}
/**
* A retry no sooner than the provider said to come back.
*/
src/bots-queue/run-bot-reply.ts
+2
View File
@@ -1,6 +1,7 @@
import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
import { AppData } from "@/src/data/app-data";
import {
ActionCheckNote,
ChatReplyNote,
CoordinatorReplyNote,
CoordinatorSoloNote,
@@ -110,6 +111,7 @@ export default async function runBotReply({
prompt,
carry_over,
ChatReplyNote,
ActionCheckNote,
workspace_id ? WorkspaceLearningNote : "",
composeComputersNote({
computers,
src/bots-queue/run-bot-task.ts
+14 -1
View File
@@ -1,5 +1,5 @@
import type { BUN_SQLITE_COBBLE_BOTS } from "@/db/types/db";
import { TaskRunNote, WorkspaceLearningNote } from "@/src/data/harness-prompts";
import { ActionCheckNote, ScheduledRunNote, TaskRunNote, WorkspaceLearningNote } from "@/src/data/harness-prompts";
import grabBotComputers from "@/src/functions/backend/db/computers/grab-bot-computers";
import grabBotPrompt from "@/src/functions/backend/db/bots/grab-bot-prompt";
import grabUserWorkspaces from "@/src/functions/backend/db/workspaces/grab-user-workspaces";
@@ -11,11 +11,13 @@ import createWorkspaceTool from "@/src/harness/tools/bot/create-workspace-tool";
import updateWorkspaceTool from "@/src/harness/tools/bot/update-workspace-tool";
import askToTakeOverTool from "@/src/harness/tools/computer/ask-to-take-over-tool";
import searchComputersTool from "@/src/harness/tools/computer/search-computers-tool";
import reportRunFailedTool from "@/src/harness/tools/bot/report-run-failed-tool";
import computerRef from "@/src/utils/computer-ref";
import composeJobBrief from "./compose-job-brief";
import grabSessionCarryOver from "./grab-session-carry-over";
import jobSessionId from "@/src/functions/backend/db/bots-queue/job-session-id";
import withTakeOverRefs from "./with-take-over-refs";
import ScheduledRunFailedError from "./scheduled-run-failed-error";
import type { BotJobContext, BotJobOutcome } from "./types";
type Params = BotJobContext & {
@@ -34,6 +36,8 @@ type Params = BotJobContext & {
* computer, that computer's tools (over MCP), until it answers. That answer is the report for the user.
* When only the user can go on (a login, a captcha), the bot asks them to
* take over (`ask_user_to_take_over`) and the report needs them.
* A scheduled run acts on its own (`ScheduledRunNote`); one that reports
* it failed (`report_run_failed`) throws, so the queue retries it.
* Every model turn is reported as a step (`report`).
*/
export default async function runBotTask({
@@ -77,6 +81,8 @@ export default async function runBotTask({
signal,
}).catch(() => "");
const take_overs: string[] = [];
const scheduled = Boolean(params.schedule_id);
let failure = null as string | null;
/**
* A take-over defaults to the computer the task runs on.
@@ -112,6 +118,7 @@ export default async function runBotTask({
}),
]
: []),
...(scheduled ? [reportRunFailedTool({ onFail: ({ reason }) => (failure = reason) })] : []),
...(computer?.tools || []),
];
@@ -128,6 +135,8 @@ export default async function runBotTask({
prompt,
carry_over,
TaskRunNote,
scheduled ? ScheduledRunNote : "",
ActionCheckNote,
workspace_id ? WorkspaceLearningNote : "",
bot.kind == "coordinator"
? `If this brief asked you to record something in a workspace (its context, or a new one), do it yourself with update_workspace or create_workspace before you report: the report alone changes nothing.`
@@ -167,6 +176,10 @@ export default async function runBotTask({
report({ ...step, text: step.text && computer ? computer.redact(step.text) : step.text }),
});
if (failure) {
throw new ScheduledRunFailedError(computer ? computer.redact(failure) : failure);
}
return withTakeOverRefs({
text: computer ? computer.redact(result.text) : result.text,
refs: take_overs,
src/bots-queue/scheduled-run-failed-error.ts
+10
View File
@@ -0,0 +1,10 @@
/**
* Thrown by a task runner when a scheduled run reported that it failed
* (`report_run_failed`). Unlike `BotJobFatalError`, the queue retries it.
*/
export default class ScheduledRunFailedError extends Error {
constructor(message: string) {
super(message);
this.name = "ScheduledRunFailedError";
}
}