Keep bot jobs and harness calls from hanging

src/bots-queue/index.ts

@@ -22,15 +22,34 @@ let timer: ReturnType<typeof setTimeout> | undefined;
  */
 export default async function botsQueue() {
     if (process.env.NODE_ENV == "production" && !process.env.MAIN_SERVER) {
+        /**
+         * Said out loud: a production-mode process without `MAIN_SERVER`
+         * answers requests but never replies, because nothing runs this
+         * queue. `bun run start` sets it.
+         */
+        bunext.bunextLog.info(
+            `Bots queue is off in this process: neither MAIN_SERVER nor development mode is set.`,
+        );
         return;
     }
 
     bunext.bunextLog.watch(`Running Bots Queue ...`);
 
-    const released = await releaseWorkerBotJobs({ worker });
+    /**
+     * Releasing the jobs a restart cut off must never stop the queue: if
+     * it fails, polling still starts, and the lock timeout releases them.
+     */
+    try {
+        const released = await releaseWorkerBotJobs({ worker });
 
-    if (released) {
-        bunext.bunextLog.info(`Bots queue: released ${released} cut-off job(s)`);
+        if (released) {
+            bunext.bunextLog.info(`Bots queue: released ${released} cut-off job(s)`);
+        }
+    } catch (error) {
+        await logError({
+            title: "Bots queue couldn't release cut-off jobs",
+            error,
+        }).catch(() => {});
     }
 
     setBotsQueueWaker({ waker: wake });

src/bots-queue/process-bot-job.ts

@@ -7,6 +7,7 @@ import updateBotJob from "@/src/functions/backend/db/bots-queue/update-bot-job";
 import insertBotJobUpdate from "@/src/functions/backend/db/bots-queue/insert-bot-job-update";
 import { logError } from "@/src/functions/backend/logger";
 import createKeyLock from "@/src/utils/create-key-lock";
+import withTimeout from "@/src/utils/with-timeout";
 import HarnessError from "@/src/harness/harness-error";
 import deferBotJob from "@/src/functions/backend/db/bots-queue/defer-bot-job";
 import recordCronJobFailure from "@/src/functions/backend/db/cron-jobs/record-cron-job-failure";
@@ -85,6 +86,13 @@ export default async function processBotJob({
         await pushBotNotice({ bot_id, bot: await setBotFace({ bot_id, user_id, ...face }) });
     };
     const controller = new AbortController();
+    /**
+     * The deadline is the one abort that is a failure, not a cancellation:
+     * the job's error is then the timeout's, not the "Cancelled." an
+     * aborted model request reports.
+     */
+    const timeout_message = `"${job.title || "The job"}" took too long and was stopped.`;
+    let timed_out = false;
 
     const update: BotJobContext["update"] = async (params) => {
         if (controller.signal.aborted) return false;
@@ -137,13 +145,27 @@ export default async function processBotJob({
                 status_line: job.kind == "reply" ? BotReplyJobTitle : BotTaskStatusLine,
             });
 
-            return await runner({
-                job,
-                params,
-                checkpoint: parseJobJson({ text: job.checkpoint }),
+            /**
+             * A deadline per attempt: the heartbeat keeps the lock alive
+             * for as long as this run takes, so without one a call that
+             * never answers would hold the slot forever.
+             */
+            return await withTimeout({
+                timeout_ms: AppData.BotJobAttemptTimeoutMs,
                 signal: controller.signal,
-                update,
-                report,
+                message: timeout_message,
+                on_timeout: () => {
+                    timed_out = true;
+                },
+                task: (run_signal) =>
+                    runner({
+                        job,
+                        params,
+                        checkpoint: parseJobJson({ text: job.checkpoint }),
+                        signal: run_signal,
+                        update,
+                        report,
+                    }),
             });
         };
 
@@ -229,7 +251,11 @@ export default async function processBotJob({
             return;
         }
 
-        const message = error instanceof Error ? error.message : String(error);
+        const message = timed_out
+            ? timeout_message
+            : error instanceof Error
+              ? error.message
+              : String(error);
         const tries = Number(job.attempts) || 1;
 
         const failed = await failBotJob({

src/bots-queue/push-bot-notice.ts

@@ -1,5 +1,6 @@
 import type { BUN_SQLITE_COBBLE_BOT_MESSAGES } from "@/db/types/db";
 import type { BotFaceChange } from "@/src/functions/backend/db/bots/set-bot-face";
+import { logError } from "@/src/functions/backend/logger";
 import notifyBotMessage from "@/src/push/notify-bot-message";
 import notifyBotFace from "@/src/push/notify-bot-face";
 import pushBotUpdated from "./push-bot-updated";
@@ -23,9 +24,22 @@ type Params = {
  * Pushes a new bot message to the open tabs of the user whose
  * conversation it's in, and a face change to the user it's for
  * (`pushBotUpdated`). A relayed report
- * goes to the conversation it was posted in.
+ * goes to the conversation it was posted in. Best effort: a push that
+ * fails is logged and never fails the job that made the change.
  */
 export default async function pushBotNotice({ bot_id, message, bot, relay }: Params) {
+    try {
+        await push({ bot_id, message, bot, relay });
+    } catch (error) {
+        await logError({
+            title: "Bot notice wasn't pushed",
+            error,
+            metadata: { bot_id, message_id: message?.id || null },
+        }).catch(() => {});
+    }
+}
+
+async function push({ bot_id, message, bot, relay }: Params) {
     if (message) {
         await pushToSessionUser({
             session_id: message.session_id,

src/bots-queue/push-bot-updated.ts

@@ -1,4 +1,5 @@
 import grabBotAudience from "@/src/functions/backend/db/access/grab-bot-audience";
+import { logError } from "@/src/functions/backend/logger";
 import grabActiveBotTask from "@/src/functions/backend/db/bots/grab-active-bot-task";
 import grabBotFace from "@/src/functions/backend/db/bots/grab-bot-face";
 import { sendToUser } from "@/src/websocket/(functions)/send-to-user";
@@ -29,7 +30,8 @@ type Params = {
 /**
  * Pushes `server:bot-updated` to one user, or everyone who sees the bot
  * (`grabBotAudience`), each with their own face, status line and active
- * task with it, never another user's.
+ * task with it, never another user's. Best effort: a push that fails is
+ * logged and never fails the change that caused it.
  */
 export default async function pushBotUpdated({
     bot_id,
@@ -40,6 +42,24 @@ export default async function pushBotUpdated({
 }: Params) {
     if (!bot_id) return;
 
+    try {
+        await push({ bot_id, user_id, users, computer_id, workspace_ids });
+    } catch (error) {
+        await logError({
+            title: "Bot update wasn't pushed",
+            error,
+            metadata: { bot_id },
+        }).catch(() => {});
+    }
+}
+
+async function push({
+    bot_id,
+    user_id,
+    users,
+    computer_id,
+    workspace_ids,
+}: Params) {
     const audience = user_id ? [user_id] : await grabBotAudience({ bot_id });
 
     for (const viewer_id of audience) {