Add workspaces, computers, bot memory and the bots queue

- Schema: workspaces (+ workspace_bots, workspace_computers), containers, computers, bot_computers, bots.memory, workspaces.system_prompt, and bots_queue as a bun-sqlite isQueue table - Main computer (and its container record) ensured with the coordinator on signup, seed and /admin/bots; the coordinator's access to it is locked - Effective computer access: direct grants plus workspace computers - Full bot prompt: core instructions, workspace prompts, then memory; bots can rewrite their memory and system prompt - Queue helpers on the native BunSQLite.queue API, plus a handler that heartbeats, serializes per computer, recovers after restarts, reports lost jobs, supports cancelling and pushes outcomes over the WebSocket - Enable SQLite foreign keys at startup so schema cascades fire - Prune finished jobs daily Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-18 18:33:54 +01:00
parent 694c83346f
commit d49f524fe8
72 changed files with 2961 additions and 31 deletions
@@ -0,0 +1,41 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+import { BotJobCancelledError } from "@/src/dict/bot-queue-dict";
+import AppError from "@/src/utils/app-error";
+
+type Params = {
+    user_id: number | string;
+    job_id: number | string;
+};
+
+/**
+ * Cancels one of the user's pending or running jobs. It's marked failed
+ * with `BotJobCancelledError`; a running handler finds out on its next
+ * `updateBotJob` (which returns false) and stops. Its task is closed.
+ */
+export default async function cancelBotJob({ user_id, job_id }: Params) {
+    const now = Date.now();
+
+    const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOTS_QUEUE>({
+        sql: `UPDATE bots_queue SET status = 'failed', last_error = ?, locked_at = NULL, locked_by = NULL, finished_at = ?, updated_at = ?
+            WHERE id = ? AND status IN ('pending', 'processing')
+            AND bot_id IN (SELECT id FROM bots WHERE user_id = ?)
+            RETURNING *`,
+        values: [BotJobCancelledError, now, now, job_id, user_id],
+    });
+
+    const job = res.payload?.[0];
+
+    if (!job?.id) {
+        throw new AppError(`No pending or running job to cancel.`);
+    }
+
+    if (job.task_id) {
+        await BunSQLite.sql({
+            sql: `UPDATE bot_tasks SET status = 'done', updated_at = ? WHERE id = ?`,
+            values: [now, job.task_id],
+        });
+    }
+
+    return job;
+}
@@ -0,0 +1,51 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type {
+    BUN_SQLITE_COBBLE_BOTS_QUEUE,
+    BunSQLiteTables,
+} from "@/db/types/db";
+
+type Params = {
+    /**
+     * Unique id of the handler claiming, e.g. `main:<pid>`. Every later
+     * update, completion or failure must pass the same id.
+     */
+    worker: string;
+    limit?: number;
+};
+
+/**
+ * Claims the next pending jobs for a queue handler. Jobs whose handler
+ * stopped sending updates are released back to the queue first, so a lost
+ * job is picked up again (resume from its `checkpoint`).
+ */
+export default async function claimBotJobs({ worker, limit = 1 }: Params) {
+    const res = await BunSQLite.queue.claim<
+        BUN_SQLITE_COBBLE_BOTS_QUEUE,
+        (typeof BunSQLiteTables)[number]
+    >({
+        table: "bots_queue",
+        worker,
+        limit,
+    });
+
+    if (!res.success) {
+        throw new Error(`Couldn't claim jobs: ${res.msg || res.error}`);
+    }
+
+    const jobs = res.payload || [];
+
+    for (const job of jobs) {
+        if (job.id && !job.started_at) {
+            const now = Date.now();
+
+            await BunSQLite.sql({
+                sql: `UPDATE bots_queue SET started_at = ? WHERE id = ?`,
+                values: [now, job.id],
+            });
+
+            job.started_at = now;
+        }
+    }
+
+    return jobs;
+}
@@ -0,0 +1,80 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import AppError from "@/src/utils/app-error";
+import insertBotMessage from "../bots/insert-bot-message";
+import setBotFace from "../bots/set-bot-face";
+import countOpenBotJobs from "./count-open-bot-jobs";
+import grabLockedBotJob from "./grab-locked-bot-job";
+
+type Params = {
+    job_id: number | string;
+    worker: string;
+    /**
+     * The completion message for the user (markdown). Also kept as the
+     * job's `result`.
+     */
+    result: string;
+    /**
+     * Flag the message as needing the user.
+     */
+    needs_you?: boolean;
+};
+
+/**
+ * Marks a job done, closes its task and sends the completion message back
+ * to the user in the bot's conversation. The bot shows done (or needs-you)
+ * unless it has other jobs open. Throws if this handler no longer holds
+ * the job.
+ */
+export default async function completeBotJob({
+    job_id,
+    worker,
+    result,
+    needs_you,
+}: Params) {
+    const job = await grabLockedBotJob({ job_id, worker });
+
+    if (!job?.id || !job.bot_id) {
+        throw new AppError(`This job is no longer held by ${worker}.`);
+    }
+
+    const res = await BunSQLite.queue.complete({
+        table: "bots_queue",
+        id: job.id,
+    });
+
+    if (!res.success) {
+        throw new Error(`Couldn't complete the job: ${res.msg || res.error}`);
+    }
+
+    const now = Date.now();
+
+    await BunSQLite.sql({
+        sql: `UPDATE bots_queue SET result = ?, finished_at = ? WHERE id = ?`,
+        values: [result, now, job.id],
+    });
+
+    if (job.task_id) {
+        await BunSQLite.sql({
+            sql: `UPDATE bot_tasks SET status = 'done', updated_at = ? WHERE id = ?`,
+            values: [now, job.task_id],
+        });
+    }
+
+    const message = await insertBotMessage({
+        bot_id: job.bot_id,
+        text: result,
+        needs_you,
+    });
+
+    const bot = needs_you
+        ? await setBotFace({
+              bot_id: job.bot_id,
+              face: "needs-you",
+              status_line: "Waiting on you",
+          })
+        : (await countOpenBotJobs({ bot_id: job.bot_id }))
+          ? null
+          : await setBotFace({ bot_id: job.bot_id, face: "done" });
+
+    return { message, bot };
+}
@@ -0,0 +1,17 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+
+type Params = {
+    bot_id: number | string;
+};
+
+/**
+ * How many of a bot's jobs are still pending or processing.
+ */
+export default async function countOpenBotJobs({ bot_id }: Params) {
+    const res = await BunSQLite.sql<{ open: number }>({
+        sql: `SELECT COUNT(*) AS open FROM bots_queue WHERE bot_id = ? AND status IN ('pending', 'processing')`,
+        values: [bot_id],
+    });
+
+    return Number(res.payload?.[0]?.open || 0);
+}
@@ -0,0 +1,148 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type {
+    BUN_SQLITE_COBBLE_BOT_TASKS,
+    BUN_SQLITE_COBBLE_BOTS_QUEUE,
+    BunSQLiteTables,
+} from "@/db/types/db";
+import AppError from "@/src/utils/app-error";
+import grabUserBot from "../bots/grab-user-bot";
+import grabBotComputers from "../computers/grab-bot-computers";
+
+type Params = {
+    user_id: number | string;
+    bot_id: number | string;
+    title: string;
+    /**
+     * The brief the bot writes for itself (markdown): what to do and how
+     * it knows it's done.
+     */
+    instructions: string;
+    params?: Record<string, unknown>;
+    /**
+     * The user message that asked for this work.
+     */
+    message_id?: number | string;
+    /**
+     * The task for the working-on panel. One is created from `title` when
+     * not given.
+     */
+    task_id?: number | string;
+    /**
+     * Must be one of the bot's computers (direct or through a workspace).
+     */
+    computer_id?: number | string;
+    /**
+     * Milliseconds to hold the job back before it can be claimed.
+     */
+    delay?: number;
+};
+
+/**
+ * Queues long-running work for one of the user's bots and returns the job
+ * id. The bot replies to the user straight away (that it's on it); the
+ * queue handler does the work and `completeBotJob` posts the result. Not
+ * every message needs a job; quick answers are replied to directly.
+ */
+export default async function enqueueBotJob({
+    user_id,
+    bot_id,
+    title,
+    instructions,
+    params,
+    message_id,
+    task_id,
+    computer_id,
+    delay,
+}: Params) {
+    const bot = await grabUserBot({ bot_id, user_id });
+
+    if (!bot?.id) {
+        throw new AppError(`Bot not found.`);
+    }
+
+    if (message_id && !(await belongsToBot({ table: "bot_messages", id: message_id, bot_id: bot.id }))) {
+        throw new AppError(`Message not found.`);
+    }
+
+    if (task_id && !(await belongsToBot({ table: "bot_tasks", id: task_id, bot_id: bot.id }))) {
+        throw new AppError(`Task not found.`);
+    }
+
+    if (computer_id) {
+        const computers = await grabBotComputers({ bot_id: bot.id, user_id });
+
+        if (!computers.some((c) => String(c.id) == String(computer_id))) {
+            throw new AppError(`This bot has no access to that computer.`);
+        }
+    }
+
+    const final_task_id = task_id || (await createJobTask({ bot_id: bot.id, title }));
+
+    const res = await BunSQLite.queue.add<
+        BUN_SQLITE_COBBLE_BOTS_QUEUE,
+        (typeof BunSQLiteTables)[number]
+    >({
+        table: "bots_queue",
+        data: [
+            {
+                bot_id: bot.id,
+                title,
+                instructions,
+                params: params ? JSON.stringify(params) : undefined,
+                message_id: message_id ? Number(message_id) : undefined,
+                task_id: Number(final_task_id),
+                computer_id: computer_id ? Number(computer_id) : undefined,
+            },
+        ],
+        delay,
+    });
+
+    const job_id = res.singleRes?.id || res.postInsertReturn?.insertId;
+
+    if (!res.success || !job_id) {
+        throw new Error(`Couldn't queue the job: ${res.msg || res.error}`);
+    }
+
+    return job_id;
+}
+
+async function belongsToBot({
+    table,
+    id,
+    bot_id,
+}: {
+    table: "bot_messages" | "bot_tasks";
+    id: number | string;
+    bot_id: number;
+}) {
+    const res = await BunSQLite.sql({
+        sql: `SELECT id FROM ${table} WHERE id = ? AND bot_id = ? LIMIT 1`,
+        values: [id, bot_id],
+    });
+
+    return Boolean(res.payload?.[0]);
+}
+
+async function createJobTask({
+    bot_id,
+    title,
+}: {
+    bot_id: number;
+    title: string;
+}) {
+    const res = await BunSQLite.insert<
+        BUN_SQLITE_COBBLE_BOT_TASKS,
+        (typeof BunSQLiteTables)[number]
+    >({
+        table: "bot_tasks",
+        data: [{ bot_id, title, status: "active" }],
+    });
+
+    const task_id = res.singleRes?.id || res.postInsertReturn?.insertId;
+
+    if (!res.success || !task_id) {
+        throw new Error(`Couldn't create the job's task: ${res.msg}`);
+    }
+
+    return task_id;
+}
@@ -0,0 +1,72 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+import { AppData } from "@/src/data/app-data";
+import AppError from "@/src/utils/app-error";
+import grabLockedBotJob from "./grab-locked-bot-job";
+import notifyBotJobFailed from "./notify-bot-job-failed";
+
+type Params = {
+    job_id: number | string;
+    worker: string;
+    error: string;
+    /**
+     * Retrying won't help (bad parameters, missing access): fail now
+     * instead of using the remaining attempts.
+     */
+    fatal?: boolean;
+    /**
+     * What to tell the user if the job is out of attempts. Defaults to a
+     * short note with the job title and error.
+     */
+    failure_message?: string;
+};
+
+/**
+ * Records a failed attempt. The job goes back to the queue with a backoff
+ * delay, or, once out of attempts (or `fatal`), is marked failed and the
+ * bot tells the user it needs them. Returns the job's new status and the
+ * failure notice, if one was sent.
+ */
+export default async function failBotJob({
+    job_id,
+    worker,
+    error,
+    fatal,
+    failure_message,
+}: Params) {
+    const job = await grabLockedBotJob({ job_id, worker });
+
+    if (!job?.id || !job.bot_id) {
+        throw new AppError(`This job is no longer held by ${worker}.`);
+    }
+
+    if (fatal) {
+        await BunSQLite.sql({
+            sql: `UPDATE bots_queue SET attempts = MAX(attempts, ?) WHERE id = ?`,
+            values: [AppData.BotQueueMaxAttempts, job.id],
+        });
+    }
+
+    const res = await BunSQLite.queue.fail<BUN_SQLITE_COBBLE_BOTS_QUEUE>({
+        table: "bots_queue",
+        id: job.id,
+        error,
+    });
+
+    if (!res.success || !res.singleRes) {
+        throw new Error(`Couldn't fail the job: ${res.msg || res.error}`);
+    }
+
+    const status = res.singleRes.status;
+
+    if (status != "failed") {
+        return { status, notice: null };
+    }
+
+    const notice = await notifyBotJobFailed({
+        job: res.singleRes,
+        failure_message,
+    });
+
+    return { status, notice };
+}
@@ -0,0 +1,23 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+
+type Params = {
+    job_id: number | string;
+    worker: string;
+};
+
+/**
+ * A job only while the given handler still holds it. Null once its lock
+ * timed out and it went back to the queue (or another handler took it).
+ */
+export default async function grabLockedBotJob({
+    job_id,
+    worker,
+}: Params): Promise<BUN_SQLITE_COBBLE_BOTS_QUEUE | null> {
+    const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOTS_QUEUE>({
+        sql: `SELECT * FROM bots_queue WHERE id = ? AND status = 'processing' AND locked_by = ? LIMIT 1`,
+        values: [job_id, worker],
+    });
+
+    return res.payload?.[0] || null;
+}
@@ -0,0 +1,15 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+
+/**
+ * Failed jobs the user hasn't been told about: the ones bun-sqlite failed
+ * itself after a lock timeout on their last attempt. Jobs failed through
+ * `failBotJob` or cancelled already have `finished_at`.
+ */
+export default async function grabUnreportedFailedBotJobs() {
+    const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOTS_QUEUE>({
+        sql: `SELECT * FROM bots_queue WHERE status = 'failed' AND finished_at IS NULL ORDER BY id ASC`,
+    });
+
+    return res.payload || [];
+}
@@ -0,0 +1,46 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+import insertBotMessage from "../bots/insert-bot-message";
+import setBotFace from "../bots/set-bot-face";
+
+type Params = {
+    job: BUN_SQLITE_COBBLE_BOTS_QUEUE;
+    failure_message?: string;
+};
+
+/**
+ * Wraps up a job that is out of attempts: stamps `finished_at` (so it's
+ * reported once), tells the user in the bot's conversation and sets the
+ * bot to needs-you. Its task stays active, since the work isn't done.
+ */
+export default async function notifyBotJobFailed({
+    job,
+    failure_message,
+}: Params) {
+    if (!job.id || !job.bot_id) {
+        throw new Error(`Job has no id or bot.`);
+    }
+
+    await BunSQLite.sql({
+        sql: `UPDATE bots_queue SET finished_at = ? WHERE id = ?`,
+        values: [Date.now(), job.id],
+    });
+
+    const message = await insertBotMessage({
+        bot_id: job.bot_id,
+        text:
+            failure_message ||
+            [`I couldn't finish "${job.title || "the job"}".`, job.last_error]
+                .filter(Boolean)
+                .join(" "),
+        needs_you: true,
+    });
+
+    const bot = await setBotFace({
+        bot_id: job.bot_id,
+        face: "needs-you",
+        status_line: "Waiting on you",
+    });
+
+    return { message, bot };
+}
@@ -0,0 +1,21 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+
+type Params = {
+    worker: string;
+};
+
+/**
+ * Puts the jobs a handler was running back in the queue straight away,
+ * instead of waiting out the lock timeout. For startup, after a restart
+ * cut those jobs off. The cut-off attempt isn't counted.
+ */
+export default async function releaseWorkerBotJobs({ worker }: Params) {
+    const now = Date.now();
+
+    const res = await BunSQLite.sql({
+        sql: `UPDATE bots_queue SET status = 'pending', attempts = MAX(attempts - 1, 0), available_at = ?, locked_at = NULL, locked_by = NULL, updated_at = ? WHERE status = 'processing' AND locked_by = ? RETURNING id`,
+        values: [now, now, worker],
+    });
+
+    return res.payload?.length || 0;
+}
@@ -0,0 +1,45 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+
+type Params = {
+    job_id: number | string;
+    worker: string;
+    /**
+     * Latest progress note (markdown), e.g. "Installed deps, running tests".
+     */
+    progress?: string;
+    /**
+     * Whatever the bot needs to resume if the job is picked up again.
+     */
+    checkpoint?: Record<string, unknown>;
+};
+
+/**
+ * The bot's "still working" update while it processes a job. Renews the
+ * lock (so the job isn't presumed lost) and saves progress and a
+ * checkpoint. Call it more often than `AppData.BotQueueLockTimeoutMs`.
+ *
+ * Returns false if this handler no longer holds the job: stop working on
+ * it, since it has gone back to the queue.
+ */
+export default async function updateBotJob({
+    job_id,
+    worker,
+    progress,
+    checkpoint,
+}: Params) {
+    const now = Date.now();
+
+    const res = await BunSQLite.sql({
+        sql: `UPDATE bots_queue SET locked_at = ?, updated_at = ?, progress = COALESCE(?, progress), checkpoint = COALESCE(?, checkpoint) WHERE id = ? AND status = 'processing' AND locked_by = ? RETURNING id`,
+        values: [
+            now,
+            now,
+            progress ?? null,
+            checkpoint ? JSON.stringify(checkpoint) : null,
+            job_id,
+            worker,
+        ],
+    });
+
+    return Boolean(res.payload?.[0]);
+}