Clear finished jobs' steps a minute after they end

Job steps (bots_queue_updates) only feed the live working row, so the
new clear-job-steps cron deletes them shortly after a job ends, however
it ended, keeping their count on bots_queue.steps for the message
details. Not done the moment a job ends: the live relay would miss the
last ones.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/cron/clear-job-steps/index.ts

@@ -0,0 +1,8 @@
+import clearFinishedJobSteps from "@/src/functions/backend/db/bots-queue/clear-finished-job-steps";
+
+/**
+ * Clears the steps of jobs that have ended (`clearFinishedJobSteps`).
+ */
+export default async function cronClearJobSteps() {
+    return await clearFinishedJobSteps();
+}

src/cron/index.ts

@@ -1,6 +1,7 @@
 import bunext from "@moduletrace/bunext";
 import cronHeartbeat from "./heartbeat";
 import pruneBotsQueue from "./prune-bots-queue";
+import cronClearJobSteps from "./clear-job-steps";
 import pruneBotHookEvents from "./prune-bot-hook-events";
 import cronPruneChatrooms from "./prune-chatrooms";
 import syncNotifyWatchers from "./sync-notify-watchers";
@@ -31,6 +32,13 @@ export default function cron() {
         task: pruneBotsQueue,
     });
 
+    registerCronJob({
+        title: "clear-job-steps",
+        schedule: "*/5 * * * *",
+        task: cronClearJobSteps,
+        quiet: true,
+    });
+
     registerCronJob({
         title: "prune-bot-hook-events",
         schedule: "@daily",

src/data/app-data.ts

@@ -414,6 +414,12 @@ export const AppData = {
      * working-on panel shows.
      */
     BotJobUpdatesShown: 12,
+    /**
+     * How long a finished job's steps stay before they're cleared
+     * (`clear-job-steps` cron). Long enough for every process's live relay
+     * to have pushed the last ones (`LiveRelayPollMs`).
+     */
+    BotJobStepsKeepMs: 60 * 1000,
     /**
      * How often each server process reads new bot messages, job steps and
      * bot changes from the database for its own sockets (`liveRelay`).

src/functions/backend/db/bots-queue/clear-finished-job-steps-sql.test.ts

@@ -0,0 +1,99 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+import clearFinishedJobStepsSql from "./clear-finished-job-steps-sql";
+
+/**
+ * Clearing a finished job's steps must keep their count on the job (the
+ * message details show it) and never touch a job that can still run: its
+ * working row reads them.
+ */
+function setup() {
+    const db = new Database(":memory:");
+
+    db.run(
+        `CREATE TABLE bots_queue (id INTEGER PRIMARY KEY, status TEXT, finished_at INTEGER, updated_at INTEGER, steps INTEGER)`,
+    );
+    db.run(`CREATE TABLE bots_queue_updates (id INTEGER PRIMARY KEY, bots_queue_id INTEGER)`);
+    db.run(`INSERT INTO bots_queue VALUES
+        (1, 'completed', 100, 100, NULL),
+        (2, 'failed', 100, 100, NULL),
+        (3, 'processing', NULL, 100, NULL),
+        (4, 'pending', NULL, 100, NULL),
+        (5, 'completed', 900, 900, NULL),
+        (6, 'failed', NULL, 100, NULL),
+        (7, 'completed', 100, 100, NULL)`);
+    db.run(`INSERT INTO bots_queue_updates (bots_queue_id) VALUES (1), (1), (1), (2), (3), (3), (4), (5), (6)`);
+
+    return db;
+}
+
+function run({ db, before }: { db: Database; before: number }) {
+    const { count, clear } = clearFinishedJobStepsSql({ before });
+
+    db.query(count.sql).run(...count.values);
+    db.query(clear.sql).run(...clear.values);
+}
+
+function stepsLeft({ db }: { db: Database }) {
+    return db
+        .query(`SELECT bots_queue_id AS id FROM bots_queue_updates ORDER BY id`)
+        .all()
+        .map((row) => (row as { id: number }).id);
+}
+
+function counts({ db }: { db: Database }) {
+    return Object.fromEntries(
+        db
+            .query(`SELECT id, steps FROM bots_queue ORDER BY id`)
+            .all()
+            .map((row) => [(row as { id: number }).id, (row as { steps: number | null }).steps]),
+    );
+}
+
+describe("clearFinishedJobStepsSql", () => {
+    test("clears the steps of jobs that ended before the cutoff, completed or failed", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        expect(stepsLeft({ db })).not.toContain(1);
+        expect(stepsLeft({ db })).not.toContain(2);
+    });
+
+    test("keeps how many steps each job recorded", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        expect(counts({ db })[1]).toBe(3);
+        expect(counts({ db })[2]).toBe(1);
+    });
+
+    test("never touches a job that can still run", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        expect(stepsLeft({ db })).toEqual([3, 3, 4, 5]);
+        expect(counts({ db })[3]).toBeNull();
+        expect(counts({ db })[4]).toBeNull();
+    });
+
+    test("leaves a job that ended after the cutoff for the next sweep", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        expect(counts({ db })[5]).toBeNull();
+        run({ db, before: 1000 });
+        expect(stepsLeft({ db })).not.toContain(5);
+        expect(counts({ db })[5]).toBe(1);
+    });
+
+    test("a job lost without a finish time is still cleared by when it last changed", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        expect(stepsLeft({ db })).not.toContain(6);
+        expect(counts({ db })[6]).toBe(1);
+    });
+
+    test("a job with no steps keeps a null count, and a second sweep changes nothing", () => {
+        const db = setup();
+        run({ db, before: 500 });
+        run({ db, before: 500 });
+        expect(counts({ db })[7]).toBeNull();
+        expect(counts({ db })[1]).toBe(3);
+    });
+});

src/functions/backend/db/bots-queue/clear-finished-job-steps-sql.ts

@@ -0,0 +1,29 @@
+type Params = {
+    /**
+     * Jobs that ended before this (unix ms) have their steps cleared.
+     */
+    before: number;
+};
+
+/**
+ * The two statements that clear the steps (`bots_queue_updates`) of jobs
+ * that ended before `before`: first each job keeps how many it recorded
+ * (`bots_queue.steps`, for the message details), then the steps go. A job
+ * that's still open, or one retrying after a failed attempt, keeps its
+ * steps. Run in this order: the count reads the rows the delete removes.
+ */
+export default function clearFinishedJobStepsSql({ before }: Params) {
+    const finished = `SELECT id FROM bots_queue WHERE status IN ('completed', 'failed') AND COALESCE(finished_at, updated_at) < ?`;
+
+    return {
+        count: {
+            sql: `UPDATE bots_queue SET steps = COALESCE(steps, 0) + (SELECT COUNT(*) FROM bots_queue_updates u WHERE u.bots_queue_id = bots_queue.id)
+                WHERE id IN (${finished}) AND EXISTS (SELECT 1 FROM bots_queue_updates u WHERE u.bots_queue_id = bots_queue.id)`,
+            values: [before],
+        },
+        clear: {
+            sql: `DELETE FROM bots_queue_updates WHERE bots_queue_id IN (${finished})`,
+            values: [before],
+        },
+    };
+}

src/functions/backend/db/bots-queue/clear-finished-job-steps.ts

@@ -0,0 +1,21 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import { AppData } from "@/src/data/app-data";
+import clearFinishedJobStepsSql from "./clear-finished-job-steps-sql";
+
+/**
+ * Clears the steps of jobs that ended more than `AppData.BotJobStepsKeepMs`
+ * ago, however they ended (completed, failed, cancelled, stopped, lost).
+ * Steps only feed the live working row; once the job's message is in they
+ * have no reader, and the job keeps their count. Not done the moment a job
+ * ends: steps reach the page only through the live relay, which would
+ * miss the last ones.
+ */
+export default async function clearFinishedJobSteps() {
+    const { count, clear } = clearFinishedJobStepsSql({
+        before: Date.now() - AppData.BotJobStepsKeepMs,
+    });
+
+    await BunSQLite.sql(count);
+
+    return await BunSQLite.sql(clear);
+}

src/functions/backend/db/usage/grab-message-details.ts

@@ -105,7 +105,7 @@ async function grabJob({ job_id }: { job_id: number }): Promise<MessageDetailsJo
     const row = (
         await BunSQLite.sql<JobRow>({
             sql: `SELECT q.id, q.kind, q.title, q.status, q.attempts, q.started_at, q.finished_at,
-                    (SELECT COUNT(*) FROM bots_queue_updates u WHERE u.bots_queue_id = q.id) AS steps
+                    COALESCE(q.steps, 0) + (SELECT COUNT(*) FROM bots_queue_updates u WHERE u.bots_queue_id = q.id) AS steps
                 FROM bots_queue q WHERE q.id = ? LIMIT 1`,
             values: [job_id],
         })

src/types/index.ts

@@ -1581,7 +1581,7 @@ export type MessageDetailsJob = {
     started_at: number | null;
     finished_at: number | null;
     /**
-     * How many model turns and notes it recorded (`bots_queue_updates`).
+     * How many model turns and notes it recorded (`bots_queue.steps` once cleared, plus any `bots_queue_updates` left).
      */
     steps: number;
 };