Keep every hook call's response, searchable by the user and the bot

A hook call used to record only how it was taken; its run's answer went
to the chat alone and was lost to the call once the job was pruned. Now
`bot_hook_events` keeps `response`, `response_message_id` and
`responded_at`, written on every call a run took in when that run ends
(completed, failed, lost, stopped or cancelled) by
`recordHookCallResponse` in the job helpers, and calls turn done or
failed. The Calls view searches what calls sent and what runs answered,
filters by status and pages; Show all loads one call whole. Bots get
`search_hook_calls` and `read_hook_call` for their own hooks (never in a
hook call's own run), with payloads always marked as outside data.
Calls are kept for the app setting `hook_events_keep_days` (Settings >
General, default now 30 days). The caller still only gets 202 Queued.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/pages/admin/settings/(functions)/grab-settings-server-props.ts

@@ -7,6 +7,7 @@ import type { PagePropsType } from "@/src/types";
 import type { BunextPageServerFn } from "@moduletrace/bunext/types";
 import grabRunLimitsOverview from "@/src/functions/backend/db/app-settings/grab-run-limits";
 import grabSkillLimitsOverview from "@/src/functions/backend/db/app-settings/grab-skill-limits";
+import grabHookCallRetentionOverview from "@/src/functions/backend/db/app-settings/grab-hook-events-keep-days";
 import grabStorageQuotaOverview from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview";
 import { grabSystemPromptAppendOverview } from "@/src/functions/backend/db/app-settings/grab-system-prompt-append";
 import runningVersion from "@/src/utils/running-version";
@@ -49,6 +50,7 @@ export default async function grabSettingsServerProps({ req }: Params) {
             system_prompt_append: await grabSystemPromptAppendOverview().catch(() => null),
             run_limits: await grabRunLimitsOverview().catch(() => null),
             skill_limits: await grabSkillLimitsOverview().catch(() => null),
+            hook_call_retention: await grabHookCallRetentionOverview().catch(() => null),
             running_version: runningVersion({ env: process.env }),
         },
     };

src/pages/admin/settings/general/(partials)/hook-call-retention-form.tsx

@@ -0,0 +1,110 @@
+import { useState } from "react";
+import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi";
+import type { APIResponseObject } from "@moduletrace/bunext/types";
+import Form from "@moduletrace/twui/form/Form";
+import Input from "@moduletrace/twui/form/Input";
+import { AppData } from "@/src/data/app-data";
+import type { HookCallRetentionOverview } from "@/src/types";
+import SettingsSaveBar from "../../(partials)/settings-save-bar";
+
+type Props = {
+    retention?: HookCallRetentionOverview | null;
+};
+
+/**
+ * How many days hook calls are kept, saved on its own without a reload.
+ * Zero puts the code's default back.
+ */
+export default function HookCallRetentionForm({ retention }: Props) {
+    const [overview, setOverview] = useState(retention || undefined);
+    const [dirty, setDirty] = useState(false);
+    const [keep_days, setKeepDays] = useState(String(retention?.keep_days ?? ""));
+    /**
+     * The input only reads `defaultValue` when it mounts, so discarding or
+     * saving remounts it with the saved number.
+     */
+    const [form_key, setFormKey] = useState(0);
+    const [loading, setLoading] = useState(false);
+    const [status, setStatus] = useState<{
+        msg?: string;
+        error?: boolean;
+        success?: boolean;
+    }>();
+
+    const save = async () => {
+        if (!window.confirm("Save how long hook calls are kept? Calls older than that are deleted at the next daily prune.")) return;
+
+        setLoading(true);
+        setStatus(undefined);
+
+        const res = await fetchApi<{}, APIResponseObject<HookCallRetentionOverview>>(
+            "/api/admin/settings/hook-call-retention",
+            {
+                method: "POST",
+                body: { keep_days: Number(keep_days) || 0 },
+            },
+        ).catch(() => null);
+
+        setLoading(false);
+
+        if (!res?.success || !res.singleRes) {
+            setStatus({
+                error: true,
+                msg: res?.msg || `Couldn't save it.`,
+            });
+            return;
+        }
+
+        setOverview(res.singleRes);
+        setKeepDays(String(res.singleRes.keep_days));
+        setFormKey((key) => key + 1);
+        setDirty(false);
+        setStatus({
+            success: true,
+            msg: `Saved: hook calls are kept for ${res.singleRes.keep_days} days.`,
+        });
+    };
+
+    const discard = () => {
+        setKeepDays(String(overview?.keep_days ?? ""));
+        setFormKey((key) => key + 1);
+        setDirty(false);
+        setStatus(undefined);
+    };
+
+    return (
+        <Form
+            onSubmit={(event) => {
+                event.preventDefault();
+                save().catch(() => {});
+            }}
+            className="w-full gap-3! items-stretch"
+        >
+            <Input
+                key={`keep-${form_key}`}
+                label="Keep hook calls for"
+                type="number"
+                min={0}
+                max={overview?.max}
+                step={1}
+                numberText
+                rawNumber
+                defaultValue={keep_days}
+                onChange={(event) => {
+                    setKeepDays(event.target.value);
+                    setDirty(true);
+                    setStatus(undefined);
+                }}
+                suffix={`days`}
+                info={`Each call keeps its payload (up to ${AppData.BotHookMaxPayloadBytes / 1024} KB) and its run's response, so a long time on a busy hook grows the database. 0 uses the default (${overview?.default_days ?? ""}).`}
+            />
+            <SettingsSaveBar
+                dirty={dirty}
+                loading={loading}
+                status={status}
+                save_label="Save"
+                onDiscard={discard}
+            />
+        </Form>
+    );
+}

src/pages/admin/settings/general/(sections)/hook-call-retention-section.tsx

@@ -0,0 +1,25 @@
+import { useContext } from "react";
+import { AppContext } from "@/src/pages/__root";
+import isPassThroughUser from "@/src/utils/rbac/is-pass-through-user";
+import SettingsGroup from "../../(partials)/settings-group";
+import HookCallRetentionForm from "../(partials)/hook-call-retention-form";
+
+/**
+ * How long hook calls and their responses are kept. Only the super admin
+ * sees it, since it changes every user's hooks.
+ */
+export default function HookCallRetentionSection() {
+    const { pageProps } = useContext(AppContext);
+
+    if (!isPassThroughUser({ user_types: pageProps?.user_types })) return null;
+
+    return (
+        <SettingsGroup
+            title="Hook calls"
+            description="How long every call to a hook is kept, with what it sent and what its run answered, so you and the bots can search them. Older calls are deleted once a day; the chat messages their runs posted stay."
+            id="hook-calls"
+        >
+            <HookCallRetentionForm retention={pageProps?.hook_call_retention} />
+        </SettingsGroup>
+    );
+}

src/pages/admin/settings/general/index.tsx

@@ -1,4 +1,5 @@
 import AppearanceSection from "./(sections)/appearance-section";
+import HookCallRetentionSection from "./(sections)/hook-call-retention-section";
 import InstallIconSection from "./(sections)/install-icon-section";
 import InstallNameSection from "./(sections)/install-name-section";
 import NotificationsSection from "./(sections)/notifications-section";
@@ -23,6 +24,7 @@ export default function AdminSettingsGeneralPage() {
             <PromptSection />
             <RunLimitsSection />
             <SkillLimitsSection />
+            <HookCallRetentionSection />
             <StorageSection />
             <VersionSection />
         </SettingsShell>

src/pages/api/admin/hooks/hook-event.ts

@@ -0,0 +1,34 @@
+import userAuth from "@/src/functions/backend/auth/user-auth";
+import grabBotHookEvent from "@/src/functions/backend/db/bot-hooks/grab-bot-hook-event";
+import { HookCallIdRequestSchema } from "@/src/schemas/bot-hook-schema";
+import type { BotHookCall } from "@/src/types";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * One call to one of the session user's hooks (`?event_id=`), whole: what
+ * it sent and what its run answered.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject<BotHookCall>> = async ({ req }) => {
+    try {
+        if (req.method != "GET") {
+            return { success: false, msg: `Method not allowed` };
+        }
+
+        const { user } = await userAuth({ req });
+
+        if (!user?.logged_in_status || !user.id) {
+            return { success: false, msg: `Unauthorized`, logoutUser: true };
+        }
+
+        const { event_id } = parseWithSchema({
+            schema: HookCallIdRequestSchema,
+            data: { event_id: new URL(req.url).searchParams.get("event_id") },
+        });
+
+        return { success: true, singleRes: await grabBotHookEvent({ user_id: user.id, event_id }) };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/pages/api/admin/hooks/hook-events.ts

@@ -1,14 +1,19 @@
-import type { BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS } from "@/db/types/db";
 import userAuth from "@/src/functions/backend/auth/user-auth";
 import grabBotHookEvents from "@/src/functions/backend/db/bot-hooks/grab-bot-hook-events";
+import type { BotHookEventStatus } from "@/src/dict/bot-hook-dict";
+import { SearchHookCallsRequestSchema } from "@/src/schemas/bot-hook-schema";
+import type { BotHookCall } from "@/src/types";
+import parseWithSchema from "@/src/utils/parse-with-schema";
 import safeErrorMessage from "@/src/utils/safe-error-message";
 import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
 
 /**
- * The latest calls to one of the session user's hooks (`?hook_id=`), each
- * payload cut to a preview.
+ * Calls to the session user's hooks, newest first: one hook's
+ * (`?hook_id=`) or all, found by words in what they sent or what their run
+ * answered (`?q=`), one status (`?status=`), older than a call
+ * (`?before_id=`). Payloads and responses are previews.
  */
-export const handler: BunextAPIRouteHandler<APIResponseObject<BUN_SQLITE_COBBLE_BOT_HOOK_EVENTS>> = async ({ req }) => {
+export const handler: BunextAPIRouteHandler<APIResponseObject<BotHookCall>> = async ({ req }) => {
     try {
         if (req.method != "GET") {
             return { success: false, msg: `Method not allowed` };
@@ -20,13 +25,26 @@ export const handler: BunextAPIRouteHandler<APIResponseObject<BUN_SQLITE_COBBLE_
             return { success: false, msg: `Unauthorized`, logoutUser: true };
         }
 
-        const hook_id = Number(new URL(req.url).searchParams.get("hook_id")) || undefined;
+        const params = Object.fromEntries(
+            [...new URL(req.url).searchParams.entries()].filter(([, value]) => value != ""),
+        );
 
-        if (!hook_id) {
-            return { success: false, msg: `Pick a hook.` };
-        }
+        const { hook_id, q, status, before_id, limit } = parseWithSchema({
+            schema: SearchHookCallsRequestSchema,
+            data: params,
+        });
 
-        return { success: true, payload: await grabBotHookEvents({ user_id: user.id, hook_id }) };
+        return {
+            success: true,
+            payload: await grabBotHookEvents({
+                user_id: user.id,
+                hook_id,
+                query: q,
+                status: status as BotHookEventStatus | undefined,
+                before_id,
+                limit,
+            }),
+        };
     } catch (error: unknown) {
         return { success: false, msg: safeErrorMessage({ error }) };
     }

src/pages/api/admin/settings/hook-call-retention.ts

@@ -0,0 +1,45 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import grabHookCallRetentionOverview from "@/src/functions/backend/db/app-settings/grab-hook-events-keep-days";
+import saveAppSetting from "@/src/functions/backend/db/app-settings/save-app-setting";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { HookEventsKeepDaysSettingKey } from "@/src/dict/bot-hook-dict";
+import { SaveHookCallRetentionRequestSchema } from "@/src/schemas/bot-hook-schema";
+import type { HookCallRetentionOverview } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: how many days hook calls and their responses are kept
+ * before the daily prune deletes them. Zero puts the code's default back.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject<HookCallRetentionOverview>> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") throw new AppError(`Method not allowed`);
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) throw new AppError(`Only a super admin can change how long hook calls are kept.`);
+
+        const { keep_days } = parseWithSchema({ schema: SaveHookCallRetentionRequestSchema, data: body });
+
+        await saveAppSetting({
+            key: HookEventsKeepDaysSettingKey,
+            value: keep_days > 0 ? String(keep_days) : null,
+        });
+
+        await logEvent({
+            title: "Hook call retention changed",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} set hook calls to be kept for ${keep_days || "the default number of"} days.`,
+            metadata: { keep_days },
+            ip_address: grabRequestIP(req),
+        });
+
+        return { success: true, singleRes: await grabHookCallRetentionOverview() };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};