Keep every hook call's response, searchable by the user and the bot

A hook call used to record only how it was taken; its run's answer went to the chat alone and was lost to the call once the job was pruned. Now `bot_hook_events` keeps `response`, `response_message_id` and `responded_at`, written on every call a run took in when that run ends (completed, failed, lost, stopped or cancelled) by `recordHookCallResponse` in the job helpers, and calls turn done or failed. The Calls view searches what calls sent and what runs answered, filters by status and pages; Show all loads one call whole. Bots get `search_hook_calls` and `read_hook_call` for their own hooks (never in a hook call's own run), with payloads always marked as outside data. Calls are kept for the app setting `hook_events_keep_days` (Settings > General, default now 30 days). The caller still only gets 202 Queued. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-25 17:47:28 +01:00
parent fb4c89fca1
commit 0e59559ea9
43 changed files with 1182 additions and 84 deletions
@@ -0,0 +1,33 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BotHookCall } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+
+type Params = {
+    user_id: number | string;
+    event_id: number | string;
+    /**
+     * Only a call to this bot's hooks.
+     */
+    bot_id?: number | string | null;
+};
+
+/**
+ * One call to one of the user's hooks, whole: what it sent and what its
+ * run answered. Throws when it isn't theirs (or that bot's).
+ */
+export default async function grabBotHookEvent({ user_id, event_id, bot_id }: Params) {
+    const res = await BunSQLite.sql<BotHookCall>({
+        sql: `SELECT e.*, h.title AS hook_title, h.bot_id, h.workspace_id
+            FROM bot_hook_events e JOIN bot_hooks h ON h.id = e.hook_id
+            WHERE e.id = ? AND h.user_id = ? AND (? IS NULL OR h.bot_id = ?)`,
+        values: [Number(event_id), user_id, bot_id ? Number(bot_id) : null, bot_id ? Number(bot_id) : null],
+    });
+
+    const call = res.payload?.[0];
+
+    if (!call?.id) {
+        throw new AppError(`Hook call #${event_id} not found.`);
+    }
+
+    return call;
+}
@@ -1,26 +1,14 @@
 import BunSQLite from "@moduletrace/bun-sqlite";
-import type { BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS } from "@/db/types/db";
-import grabUserBotHook from "./grab-user-bot-hook";
-
-type Params = {
-    user_id: number | string;
-    hook_id: number | string;
-    limit?: number;
-};
+import type { BotHookCall } from "@/src/types";
+import searchHookCallsSql, { type SearchHookCallsParams } from "./search-hook-calls-sql";

 /**
- * One of the user's hooks' latest calls, newest first, each payload cut
- * to a preview in SQL.
+ * Calls to the user's hooks, newest first: all of them, one hook's or one
+ * bot's, found by words in what they sent or what their run answered.
+ * Each payload and response is cut to a preview in SQL.
 */
-export default async function grabBotHookEvents({ user_id, hook_id, limit = 20 }: Params) {
-    const hook = await grabUserBotHook({ user_id, hook_id });
-
-    const res = await BunSQLite.sql<BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS>({
-        sql: `SELECT id, created_at, hook_id, job_id, substr(payload, 1, 400) AS payload, content_type,
-                source_ip, idempotency_key, status, error
-            FROM bot_hook_events WHERE hook_id = ? ORDER BY id DESC LIMIT ?`,
-        values: [hook.id!, Math.min(Math.max(limit, 1), 100)],
-    });
+export default async function grabBotHookEvents(params: SearchHookCallsParams) {
+    const res = await BunSQLite.sql<BotHookCall>(searchHookCallsSql(params));

    return res.payload || [];
 }
@@ -0,0 +1,51 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+import recordHookCallResponseSql from "./record-hook-call-response-sql";
+
+/**
+ * A run's response lands on every call it took in, and on nothing else.
+ * Run against real SQLite, since a wrong condition would either miss the
+ * calls merged into the run or write over calls already answered.
+ */
+const db = new Database(":memory:");
+
+db.run(`CREATE TABLE bot_hook_events (id INTEGER PRIMARY KEY, hook_id INTEGER, job_id INTEGER, status TEXT,
+    response TEXT, response_message_id INTEGER, responded_at INTEGER, updated_at INTEGER)`);
+db.run(`INSERT INTO bot_hook_events (id, hook_id, job_id, status) VALUES
+    (1, 1, 10, 'queued'),
+    (2, 1, 10, 'merged'),
+    (3, 1, 11, 'queued'),
+    (4, 2, 10, 'queued'),
+    (5, 1, 10, 'rejected'),
+    (6, 1, 12, 'done')`);
+
+function record(params: Parameters<typeof recordHookCallResponseSql>[0]) {
+    const { sql, values } = recordHookCallResponseSql(params);
+
+    return (db.query(sql).all(...values) as { id: number }[]).map((row) => row.id);
+}
+
+function row(id: number) {
+    return db.query(`SELECT * FROM bot_hook_events WHERE id = ?`).get(id) as Record<string, unknown>;
+}
+
+describe("recordHookCallResponseSql", () => {
+    test("answers every call the run took in", () => {
+        expect(record({ hook_id: 1, job_id: 10, status: "done", response: "Done it", message_id: 7, now: 42 })).toEqual([1, 2]);
+        expect(row(1)).toMatchObject({ status: "done", response: "Done it", response_message_id: 7, responded_at: 42 });
+        expect(row(2)).toMatchObject({ status: "done", response: "Done it" });
+    });
+
+    test("leaves other runs, other hooks and settled calls alone", () => {
+        expect(row(3)).toMatchObject({ status: "queued", response: null });
+        expect(row(4)).toMatchObject({ status: "queued", response: null });
+        expect(row(5)).toMatchObject({ status: "rejected", response: null });
+        expect(record({ hook_id: 1, job_id: 12, status: "failed", response: "Nope", now: 43 })).toEqual([]);
+        expect(row(6)).toMatchObject({ status: "done", response: null });
+    });
+
+    test("a failed run keeps why", () => {
+        expect(record({ hook_id: 1, job_id: 11, status: "failed", response: "Timed out", now: 44 })).toEqual([3]);
+        expect(row(3)).toMatchObject({ status: "failed", response: "Timed out", response_message_id: null });
+    });
+});
@@ -0,0 +1,25 @@
+type Params = {
+    hook_id: number;
+    /**
+     * The run that ended.
+     */
+    job_id: number;
+    status: "done" | "failed";
+    response: string;
+    message_id?: number | null;
+    now: number;
+};
+
+/**
+ * Writes a run's end onto every call it took in (the one that queued it
+ * and any merged into it): answered or failed, with its response. Only
+ * calls still waiting on it change, so a call already answered never is
+ * written over.
+ */
+export default function recordHookCallResponseSql({ hook_id, job_id, status, response, message_id, now }: Params) {
+    return {
+        sql: `UPDATE bot_hook_events SET status = ?, response = ?, response_message_id = ?, responded_at = ?, updated_at = ?
+            WHERE hook_id = ? AND job_id = ? AND status IN ('queued', 'merged') RETURNING id`,
+        values: [status, response, message_id ?? null, now, now, hook_id, job_id] as (string | number | null)[],
+    };
+}
@@ -0,0 +1,47 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import type { BUN_SQLITE_COBBLE_BOTS_QUEUE } from "@/db/types/db";
+import parseJobJson from "@/src/bots-queue/parse-job-json";
+import { AppData } from "@/src/data/app-data";
+import recordHookCallResponseSql from "./record-hook-call-response-sql";
+
+type Params = {
+    /**
+     * The job that ended. Only a hook call's run (`params.hook_id`) has
+     * calls to answer; any other job is left alone.
+     */
+    job: BUN_SQLITE_COBBLE_BOTS_QUEUE;
+    status: "done" | "failed";
+    /**
+     * What the run answered, or why it failed. Cut to
+     * `AppData.BotHookMaxPayloadBytes`.
+     */
+    response: string;
+    /**
+     * The chat message the run's end posted, if it posted one.
+     */
+    message_id?: number | string | null;
+    now?: number;
+};
+
+/**
+ * Keeps a hook run's end on the calls it took in, so they can be found
+ * later by what was sent or answered, after the job itself is pruned.
+ * Called wherever a job ends (completed, failed for good, lost, stopped,
+ * cancelled). Never throws: the job has ended either way.
+ */
+export default async function recordHookCallResponse({ job, status, response, message_id, now = Date.now() }: Params) {
+    const hook_id = Number(parseJobJson({ text: job.params })?.hook_id) || 0;
+
+    if (!hook_id || !job.id) return;
+
+    await BunSQLite.sql(
+        recordHookCallResponseSql({
+            hook_id,
+            job_id: Number(job.id),
+            status,
+            response: response.slice(0, AppData.BotHookMaxPayloadBytes),
+            message_id: Number(message_id) || null,
+            now,
+        }),
+    ).catch(() => {});
+}
@@ -0,0 +1,59 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+import searchHookCallsSql, { type SearchHookCallsParams } from "./search-hook-calls-sql";
+
+/**
+ * Searching hook calls only ever reads the user's own hooks, and finds a
+ * call by what it sent or what its run answered. Run against real SQLite.
+ */
+const db = new Database(":memory:");
+
+db.run(`CREATE TABLE bot_hooks (id INTEGER PRIMARY KEY, user_id INTEGER, bot_id INTEGER, workspace_id INTEGER, title TEXT)`);
+db.run(`CREATE TABLE bot_hook_events (id INTEGER PRIMARY KEY, created_at INTEGER, hook_id INTEGER, job_id INTEGER,
+    payload TEXT, content_type TEXT, source_ip TEXT, idempotency_key TEXT, status TEXT, error TEXT, response TEXT,
+    response_message_id INTEGER, responded_at INTEGER)`);
+db.run(`INSERT INTO bot_hooks VALUES (1, 1, 10, NULL, 'Orders'), (2, 1, 11, NULL, 'Builds'), (3, 2, 10, NULL, 'Theirs')`);
+db.run(`INSERT INTO bot_hook_events (id, hook_id, payload, status, response, error) VALUES
+    (1, 1, '{"order":"A-100"}', 'done', 'Shipped A-100', NULL),
+    (2, 1, '{"order":"B-200"}', 'failed', 'Timed out', NULL),
+    (3, 2, '{"build":"green"}', 'done', 'All 100% passing', NULL),
+    (4, 3, '{"order":"A-100"}', 'done', 'Not yours', NULL),
+    (5, 1, 'bad', 'rejected', NULL, 'Bad or missing signature')`);
+
+function ids(params: Partial<SearchHookCallsParams>) {
+    const { sql, values } = searchHookCallsSql({ user_id: 1, ...params });
+
+    return (db.query(sql).all(...values) as { id: number }[]).map((row) => row.id);
+}
+
+describe("searchHookCallsSql", () => {
+    test("lists only the user's calls, newest first", () => {
+        expect(ids({})).toEqual([5, 3, 2, 1]);
+        expect(ids({ user_id: 2 })).toEqual([4]);
+    });
+
+    test("finds a call by its payload, response or error", () => {
+        expect(ids({ query: "A-100" })).toEqual([1]);
+        expect(ids({ query: "timed OUT" })).toEqual([2]);
+        expect(ids({ query: "signature" })).toEqual([5]);
+    });
+
+    test("matches % and _ literally", () => {
+        expect(ids({ query: "100%" })).toEqual([3]);
+        expect(ids({ query: "_" })).toEqual([]);
+    });
+
+    test("filters by hook, bot and status, and pages", () => {
+        expect(ids({ hook_id: 2 })).toEqual([3]);
+        expect(ids({ bot_id: 10 })).toEqual([5, 2, 1]);
+        expect(ids({ status: "done" })).toEqual([3, 1]);
+        expect(ids({ before_id: 3, limit: 1 })).toEqual([2]);
+    });
+
+    test("cuts payloads and responses to previews", () => {
+        const { sql, values } = searchHookCallsSql({ user_id: 1, hook_id: 2, preview_chars: 3 });
+        const [row] = db.query(sql).all(...values) as { payload: string; response: string; hook_title: string }[];
+
+        expect(row).toMatchObject({ payload: '{"b', response: "All", hook_title: "Builds" });
+    });
+});
@@ -0,0 +1,85 @@
+import { AppData } from "@/src/data/app-data";
+import type { BotHookEventStatus } from "@/src/dict/bot-hook-dict";
+
+export type SearchHookCallsParams = {
+    /**
+     * Whose hooks: only calls to hooks this user owns are ever read.
+     */
+    user_id: number | string;
+    hook_id?: number | string | null;
+    bot_id?: number | string | null;
+    /**
+     * Words to find in what a call sent or what its run answered.
+     */
+    query?: string | null;
+    status?: BotHookEventStatus | null;
+    /**
+     * Only calls older than this one (paging).
+     */
+    before_id?: number | string | null;
+    limit?: number;
+    /**
+     * How much of each payload and response comes back.
+     */
+    preview_chars?: number;
+};
+
+/**
+ * Finds calls to the user's hooks, newest first, with each one's hook name
+ * and bot. Payloads and responses are cut in SQL, so a list never carries
+ * whole bodies. `%` and `_` in the query match only themselves.
+ */
+export default function searchHookCallsSql({
+    user_id,
+    hook_id,
+    bot_id,
+    query,
+    status,
+    before_id,
+    limit = 20,
+    preview_chars = AppData.BotHookCallPreviewChars,
+}: SearchHookCallsParams) {
+    const where = [`h.user_id = ?`];
+    const values: (string | number)[] = [preview_chars, preview_chars, user_id];
+
+    if (hook_id) {
+        where.push(`e.hook_id = ?`);
+        values.push(Number(hook_id));
+    }
+
+    if (bot_id) {
+        where.push(`h.bot_id = ?`);
+        values.push(Number(bot_id));
+    }
+
+    if (status) {
+        where.push(`e.status = ?`);
+        values.push(status);
+    }
+
+    if (before_id) {
+        where.push(`e.id < ?`);
+        values.push(Number(before_id));
+    }
+
+    const search = query?.trim();
+
+    if (search) {
+        const like = `%${search.replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
+
+        where.push(`(e.payload LIKE ? ESCAPE '\\' OR e.response LIKE ? ESCAPE '\\' OR e.error LIKE ? ESCAPE '\\')`);
+        values.push(like, like, like);
+    }
+
+    values.push(Math.min(Math.max(Math.floor(limit) || 20, 1), 100));
+
+    return {
+        sql: `SELECT e.id, e.created_at, e.hook_id, e.job_id, substr(e.payload, 1, ?) AS payload, e.content_type,
+                e.source_ip, e.idempotency_key, e.status, e.error, substr(e.response, 1, ?) AS response,
+                e.response_message_id, e.responded_at, h.title AS hook_title, h.bot_id, h.workspace_id
+            FROM bot_hook_events e JOIN bot_hooks h ON h.id = e.hook_id
+            WHERE ${where.join(" AND ")}
+            ORDER BY e.id DESC LIMIT ?`,
+        values,
+    };
+}