Keep logs for a set number of days

Logs grew forever. A daily prune-logs cron now deletes entries older
than the app setting logs_keep_days (Settings > General, super admin;
default AppData.LogsKeepDays, 90), with an index on logs.created_at.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/cron/index.ts

@@ -4,6 +4,7 @@ import pruneBotsQueue from "./prune-bots-queue";
 import cronClearJobSteps from "./clear-job-steps";
 import pruneBotHookEvents from "./prune-bot-hook-events";
 import cronPruneChatrooms from "./prune-chatrooms";
+import pruneLogs from "./prune-logs";
 import syncNotifyWatchers from "./sync-notify-watchers";
 import cronSyncHarnessModels from "./sync-harness-models";
 import syncComputers from "./sync-computers";
@@ -51,6 +52,12 @@ export default function cron() {
         task: cronPruneChatrooms,
     });
 
+    registerCronJob({
+        title: "prune-logs",
+        schedule: "@daily",
+        task: pruneLogs,
+    });
+
     registerCronJob({
         title: "sync-notify-watchers",
         schedule: "*/5 * * * *",

src/cron/prune-logs/index.ts

@@ -0,0 +1,15 @@
+import BunSQLite from "@moduletrace/bun-sqlite";
+import { grabLogsKeepDays } from "@/src/functions/backend/db/app-settings/grab-logs-keep-days";
+
+/**
+ * Deletes logs older than the install's retention (`grabLogsKeepDays`).
+ * Nothing points at a log, so no children go with them.
+ */
+export default async function pruneLogs() {
+    const cutoff = Date.now() - (await grabLogsKeepDays()) * 24 * 60 * 60 * 1000;
+
+    return await BunSQLite.sql({
+        sql: `DELETE FROM logs WHERE created_at < ? RETURNING id`,
+        values: [cutoff],
+    });
+}

src/data/app-data.ts

@@ -278,6 +278,13 @@ export const AppData = {
     BotHookMaxEventsPerRun: 20,
     BotHookEventsKeepDays: 30,
     BotHookEventsKeepDaysMax: 3650,
+    /**
+     * Logs (Activity) are deleted after this many days by the daily
+     * `prune-logs` cron: the default, with the most an install may set
+     * (`logs_keep_days`, Settings > General).
+     */
+    LogsKeepDays: 90,
+    LogsKeepDaysMax: 3650,
     /**
      * How many hook calls a list loads at once, and how much of each call's
      * payload and response a list shows (the whole of both opens on one).

src/dict/log-categories.ts

@@ -23,4 +23,10 @@ export const LogCategories = [
         title: "Error",
         value: "error",
     },
-] as const;
+] as const;
+
+/**
+ * The app setting (Settings > General) for how many days logs are kept,
+ * else `AppData.LogsKeepDays`.
+ */
+export const LogsKeepDaysSettingKey = "logs_keep_days";

src/functions/backend/db/app-settings/grab-logs-keep-days.ts

@@ -0,0 +1,30 @@
+import { AppData } from "@/src/data/app-data";
+import { LogsKeepDaysSettingKey } from "@/src/dict/log-categories";
+import type { LogRetentionOverview } from "@/src/types";
+import readStepLimit from "@/src/utils/read-step-limit";
+import grabAppSetting from "./grab-app-setting";
+
+/**
+ * How many days logs are kept: the install's own number (`logs_keep_days`,
+ * Settings > General), else `AppData.LogsKeepDays`.
+ */
+export async function grabLogsKeepDays() {
+    return readStepLimit({
+        saved: await grabAppSetting({ key: LogsKeepDaysSettingKey }),
+        fallback: AppData.LogsKeepDays,
+        max: AppData.LogsKeepDaysMax,
+    });
+}
+
+/**
+ * The number with its default and ceiling, for the settings page.
+ */
+export default async function grabLogRetentionOverview() {
+    const overview: LogRetentionOverview = {
+        keep_days: await grabLogsKeepDays(),
+        default_days: AppData.LogsKeepDays,
+        max: AppData.LogsKeepDaysMax,
+    };
+
+    return overview;
+}

src/pages/admin/settings/(functions)/grab-settings-server-props.ts

@@ -8,6 +8,7 @@ import type { BunextPageServerFn } from "@moduletrace/bunext/types";
 import grabRunLimitsOverview from "@/src/functions/backend/db/app-settings/grab-run-limits";
 import grabSkillLimitsOverview from "@/src/functions/backend/db/app-settings/grab-skill-limits";
 import grabHookCallRetentionOverview from "@/src/functions/backend/db/app-settings/grab-hook-events-keep-days";
+import grabLogRetentionOverview from "@/src/functions/backend/db/app-settings/grab-logs-keep-days";
 import grabChatroomRetentionOverview from "@/src/functions/backend/db/app-settings/grab-chatroom-retention";
 import grabSessionLengthOverview from "@/src/functions/backend/db/app-settings/grab-bot-session-max-messages";
 import grabStorageQuotaOverview from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview";
@@ -53,6 +54,7 @@ export default async function grabSettingsServerProps({ req }: Params) {
             run_limits: await grabRunLimitsOverview().catch(() => null),
             skill_limits: await grabSkillLimitsOverview().catch(() => null),
             hook_call_retention: await grabHookCallRetentionOverview().catch(() => null),
+            log_retention: await grabLogRetentionOverview().catch(() => null),
             chatroom_retention: await grabChatroomRetentionOverview().catch(() => null),
             session_length: await grabSessionLengthOverview().catch(() => null),
             running_version: runningVersion({ env: process.env }),

src/pages/admin/settings/general/(partials)/log-retention-form.tsx

@@ -0,0 +1,109 @@
+import { useState } from "react";
+import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi";
+import type { APIResponseObject } from "@moduletrace/bunext/types";
+import Form from "@moduletrace/twui/form/Form";
+import Input from "@moduletrace/twui/form/Input";
+import type { LogRetentionOverview } from "@/src/types";
+import SettingsSaveBar from "../../(partials)/settings-save-bar";
+
+type Props = {
+    retention?: LogRetentionOverview | null;
+};
+
+/**
+ * How many days logs are kept, saved on its own without a reload.
+ * Zero puts the code's default back.
+ */
+export default function LogRetentionForm({ retention }: Props) {
+    const [overview, setOverview] = useState(retention || undefined);
+    const [dirty, setDirty] = useState(false);
+    const [keep_days, setKeepDays] = useState(String(retention?.keep_days ?? ""));
+    /**
+     * The input only reads `defaultValue` when it mounts, so discarding or
+     * saving remounts it with the saved number.
+     */
+    const [form_key, setFormKey] = useState(0);
+    const [loading, setLoading] = useState(false);
+    const [status, setStatus] = useState<{
+        msg?: string;
+        error?: boolean;
+        success?: boolean;
+    }>();
+
+    const save = async () => {
+        if (!window.confirm("Save how long logs are kept? Entries older than that are deleted at the next daily prune.")) return;
+
+        setLoading(true);
+        setStatus(undefined);
+
+        const res = await fetchApi<{}, APIResponseObject<LogRetentionOverview>>(
+            "/api/admin/settings/log-retention",
+            {
+                method: "POST",
+                body: { keep_days: Number(keep_days) || 0 },
+            },
+        ).catch(() => null);
+
+        setLoading(false);
+
+        if (!res?.success || !res.singleRes) {
+            setStatus({
+                error: true,
+                msg: res?.msg || `Couldn't save it.`,
+            });
+            return;
+        }
+
+        setOverview(res.singleRes);
+        setKeepDays(String(res.singleRes.keep_days));
+        setFormKey((key) => key + 1);
+        setDirty(false);
+        setStatus({
+            success: true,
+            msg: `Saved: logs are kept for ${res.singleRes.keep_days} days.`,
+        });
+    };
+
+    const discard = () => {
+        setKeepDays(String(overview?.keep_days ?? ""));
+        setFormKey((key) => key + 1);
+        setDirty(false);
+        setStatus(undefined);
+    };
+
+    return (
+        <Form
+            onSubmit={(event) => {
+                event.preventDefault();
+                save().catch(() => {});
+            }}
+            className="w-full gap-3! items-stretch"
+        >
+            <Input
+                key={`keep-${form_key}`}
+                label="Keep activity for"
+                type="number"
+                min={0}
+                max={overview?.max}
+                step={1}
+                numberText
+                rawNumber
+                defaultValue={keep_days}
+                onChange={(event) => {
+                    setKeepDays(event.target.value);
+                    setDirty(true);
+                    setStatus(undefined);
+                }}
+                suffix={`days`}
+                info={`Every cron run writes an entry, so a long time grows the database. 0 uses the default (${overview?.default_days ?? ""}).`}
+            />
+            <SettingsSaveBar
+                dirty={dirty}
+                loading={loading}
+                status={status}
+                save_label="Save"
+                onDiscard={discard}
+            />
+        </Form>
+    );
+}

src/pages/admin/settings/general/(sections)/log-retention-section.tsx

@@ -0,0 +1,25 @@
+import { useContext } from "react";
+import { AppContext } from "@/src/pages/__root";
+import isPassThroughUser from "@/src/utils/rbac/is-pass-through-user";
+import SettingsGroup from "../../(partials)/settings-group";
+import LogRetentionForm from "../(partials)/log-retention-form";
+
+/**
+ * How long logs (Activity) are kept. Only the super admin sees it, since
+ * only the super admin reads them.
+ */
+export default function LogRetentionSection() {
+    const { pageProps } = useContext(AppContext);
+
+    if (!isPassThroughUser({ user_types: pageProps?.user_types })) return null;
+
+    return (
+        <SettingsGroup
+            title="Activity log"
+            description="How long sign-ins, routine runs, errors and other events in Activity are kept. Older entries are deleted once a day."
+            id="activity-log"
+        >
+            <LogRetentionForm retention={pageProps?.log_retention} />
+        </SettingsGroup>
+    );
+}

src/pages/admin/settings/general/index.tsx

@@ -3,6 +3,7 @@ import ChatroomRetentionSection from "./(sections)/chatroom-retention-section";
 import HookCallRetentionSection from "./(sections)/hook-call-retention-section";
 import InstallIconSection from "./(sections)/install-icon-section";
 import InstallNameSection from "./(sections)/install-name-section";
+import LogRetentionSection from "./(sections)/log-retention-section";
 import NotificationsSection from "./(sections)/notifications-section";
 import PromptSection from "./(sections)/prompt-section";
 import RunLimitsSection from "./(sections)/run-limits-section";
@@ -29,6 +30,7 @@ export default function AdminSettingsGeneralPage() {
             <SkillLimitsSection />
             <HookCallRetentionSection />
             <ChatroomRetentionSection />
+            <LogRetentionSection />
             <StorageSection />
             <VersionSection />
         </SettingsShell>

src/pages/api/admin/settings/log-retention.ts

@@ -0,0 +1,45 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import grabLogRetentionOverview from "@/src/functions/backend/db/app-settings/grab-logs-keep-days";
+import saveAppSetting from "@/src/functions/backend/db/app-settings/save-app-setting";
+import logEvent, { grabRequestIP } from "@/src/functions/backend/logger";
+import { LogsKeepDaysSettingKey } from "@/src/dict/log-categories";
+import { SaveLogRetentionRequestSchema } from "@/src/schemas/log-schema";
+import type { LogRetentionOverview } from "@/src/types";
+import AppError from "@/src/utils/app-error";
+import parseWithSchema from "@/src/utils/parse-with-schema";
+import safeErrorMessage from "@/src/utils/safe-error-message";
+import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types";
+
+/**
+ * Super admin only: how many days logs are kept before the daily prune
+ * deletes them. Zero puts the code's default back.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject<LogRetentionOverview>> = async ({ req, body }) => {
+    try {
+        if (req.method != "POST") throw new AppError(`Method not allowed`);
+
+        const user = await grabSuperAdminUser({ req });
+
+        if (!user?.id) throw new AppError(`Only a super admin can change how long logs are kept.`);
+
+        const { keep_days } = parseWithSchema({ schema: SaveLogRetentionRequestSchema, data: body });
+
+        await saveAppSetting({
+            key: LogsKeepDaysSettingKey,
+            value: keep_days > 0 ? String(keep_days) : null,
+        });
+
+        await logEvent({
+            title: "Log retention changed",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} set logs to be kept for ${keep_days || "the default number of"} days.`,
+            metadata: { keep_days },
+            ip_address: grabRequestIP(req),
+        });
+
+        return { success: true, singleRes: await grabLogRetentionOverview() };
+    } catch (error: unknown) {
+        return { success: false, msg: safeErrorMessage({ error }) };
+    }
+};

src/schemas/log-schema.ts

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { AppData } from "@/src/data/app-data";
 import { LogCategories } from "@/src/dict/log-categories";
 
 const LogCategoryValues = LogCategories.map((lc) => lc.value) as [
@@ -15,3 +16,11 @@ export const ClearLogsRequestSchema = z.object({
 });
 
 export type ClearLogsRequest = z.infer<typeof ClearLogsRequestSchema>;
+
+/**
+ * How many days logs are kept (Settings > General). Zero puts the code's
+ * default back.
+ */
+export const SaveLogRetentionRequestSchema = z.object({
+    keep_days: z.coerce.number().int().min(0).max(AppData.LogsKeepDaysMax),
+});

src/types/index.ts

@@ -1117,6 +1117,15 @@ export type HookCallRetentionOverview = {
     max: number;
 };
 
+/**
+ * How long logs are kept (Settings > General, `grab-logs-keep-days.ts`).
+ */
+export type LogRetentionOverview = {
+    keep_days: number;
+    default_days: number;
+    max: number;
+};
+
 /**
  * The install's skill limits (Settings > General, `grab-skill-limits.ts`).
  */
@@ -1911,6 +1920,11 @@ export type PagePropsType = {
      * General settings page).
      */
     hook_call_retention?: HookCallRetentionOverview | null;
+    /**
+     * How many days logs are kept, with the code's default (the General
+     * settings page).
+     */
+    log_retention?: LogRetentionOverview | null;
     /**
      * How long closed chatrooms are kept and how many one place keeps, with
      * the code's defaults (the General settings page).