diff --git a/AGENTS.md b/AGENTS.md index 833adb8..1197afc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,7 +60,7 @@ Enables SQLite foreign keys (`enableForeignKeys`), `bunextInit()`, `ensureDocker - **Passwords:** bcrypt in `user_passwords` (never in `users`; blocked from CRUD via `src/dict/private-tables.ts`). - **User types** (`src/dict/user-types-dict.ts`, the only definition): `super_admin` (user #1), `admin` (only the super admin grants it), `member` (default), `read_only` (non-GET CRUD blocked), `revoked`. A user needs at least one type to sign in. - **RBAC:** `src/utils/check-user-access.ts` plus the matrix engine in `src/utils/rbac/` (`userTypeAssignment` over `UserTypeAssignmentMatrix`/`UserTypeDeletionMatrix`). -- **Audit:** `src/functions/backend/logger.ts` (`logEvent`/`logError` -> `logs`). Auth events, cron runs, every unexpected API error (`safeErrorMessage` -> `logUnexpectedError`, not for `AppError`), every failed or lost bot job. WebSocket connects are not logged. +- **Audit:** `src/functions/backend/logger.ts` (`logEvent`/`logError` -> `logs`). Auth events, cron runs, every unexpected API error (`safeErrorMessage` -> `logUnexpectedError`, not for `AppError`), every failed or lost bot job. WebSocket connects are not logged. Kept for the app setting `logs_keep_days` (default `AppData.LogsKeepDays`), deleted by the daily `prune-logs`. ### WebSocket @@ -68,7 +68,7 @@ Enables SQLite foreign keys (`enableForeignKeys`), `bunextInit()`, `ensureDocker ### Cron -`src/cron/index.ts` registers everything via `registerCronJob` (`src/cron/register-cron-job.ts`); one folder per task. In-process `Bun.cron`, UTC, never overlapping. Notable: `heartbeat`, `sync-harness-models` (hourly), `run-cron-jobs` (every minute, fires schedules), `sync-computers`, `sync-mcp-servers`, `clear-job-steps`, `prune-bots-queue`, `prune-chatrooms`. +`src/cron/index.ts` registers everything via `registerCronJob` (`src/cron/register-cron-job.ts`); one folder per task. In-process `Bun.cron`, UTC, never overlapping. Notable: `heartbeat`, `sync-harness-models` (hourly), `run-cron-jobs` (every minute, fires schedules), `sync-computers`, `sync-mcp-servers`, `clear-job-steps`, `prune-bots-queue`, `prune-chatrooms`, `prune-logs`. ### Admin CRUD API diff --git a/db/schema.ts b/db/schema.ts index b8c6005..f21f6c6 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -3532,6 +3532,10 @@ const schema: BUN_SQLITE_DatabaseSchemaType = { indexName: "idx_logs_user_id", indexTableFields: ["user_id"], }, + { + indexName: "idx_logs_created_at", + indexTableFields: ["created_at"], + }, ], }, { diff --git a/src/cron/index.ts b/src/cron/index.ts index c3377c7..7f7104c 100644 --- a/src/cron/index.ts +++ b/src/cron/index.ts @@ -4,6 +4,7 @@ import pruneBotsQueue from "./prune-bots-queue"; import cronClearJobSteps from "./clear-job-steps"; import pruneBotHookEvents from "./prune-bot-hook-events"; import cronPruneChatrooms from "./prune-chatrooms"; +import pruneLogs from "./prune-logs"; import syncNotifyWatchers from "./sync-notify-watchers"; import cronSyncHarnessModels from "./sync-harness-models"; import syncComputers from "./sync-computers"; @@ -51,6 +52,12 @@ export default function cron() { task: cronPruneChatrooms, }); + registerCronJob({ + title: "prune-logs", + schedule: "@daily", + task: pruneLogs, + }); + registerCronJob({ title: "sync-notify-watchers", schedule: "*/5 * * * *", diff --git a/src/cron/prune-logs/index.ts b/src/cron/prune-logs/index.ts new file mode 100644 index 0000000..d4269b3 --- /dev/null +++ b/src/cron/prune-logs/index.ts @@ -0,0 +1,15 @@ +import BunSQLite from "@moduletrace/bun-sqlite"; +import { grabLogsKeepDays } from "@/src/functions/backend/db/app-settings/grab-logs-keep-days"; + +/** + * Deletes logs older than the install's retention (`grabLogsKeepDays`). + * Nothing points at a log, so no children go with them. + */ +export default async function pruneLogs() { + const cutoff = Date.now() - (await grabLogsKeepDays()) * 24 * 60 * 60 * 1000; + + return await BunSQLite.sql({ + sql: `DELETE FROM logs WHERE created_at < ? RETURNING id`, + values: [cutoff], + }); +} diff --git a/src/data/app-data.ts b/src/data/app-data.ts index 354f4b2..0872b83 100644 --- a/src/data/app-data.ts +++ b/src/data/app-data.ts @@ -278,6 +278,13 @@ export const AppData = { BotHookMaxEventsPerRun: 20, BotHookEventsKeepDays: 30, BotHookEventsKeepDaysMax: 3650, + /** + * Logs (Activity) are deleted after this many days by the daily + * `prune-logs` cron: the default, with the most an install may set + * (`logs_keep_days`, Settings > General). + */ + LogsKeepDays: 90, + LogsKeepDaysMax: 3650, /** * How many hook calls a list loads at once, and how much of each call's * payload and response a list shows (the whole of both opens on one). diff --git a/src/dict/log-categories.ts b/src/dict/log-categories.ts index 7e19988..ca8d6b8 100644 --- a/src/dict/log-categories.ts +++ b/src/dict/log-categories.ts @@ -23,4 +23,10 @@ export const LogCategories = [ title: "Error", value: "error", }, -] as const; \ No newline at end of file +] as const; + +/** + * The app setting (Settings > General) for how many days logs are kept, + * else `AppData.LogsKeepDays`. + */ +export const LogsKeepDaysSettingKey = "logs_keep_days"; diff --git a/src/functions/backend/db/app-settings/grab-logs-keep-days.ts b/src/functions/backend/db/app-settings/grab-logs-keep-days.ts new file mode 100644 index 0000000..098b838 --- /dev/null +++ b/src/functions/backend/db/app-settings/grab-logs-keep-days.ts @@ -0,0 +1,30 @@ +import { AppData } from "@/src/data/app-data"; +import { LogsKeepDaysSettingKey } from "@/src/dict/log-categories"; +import type { LogRetentionOverview } from "@/src/types"; +import readStepLimit from "@/src/utils/read-step-limit"; +import grabAppSetting from "./grab-app-setting"; + +/** + * How many days logs are kept: the install's own number (`logs_keep_days`, + * Settings > General), else `AppData.LogsKeepDays`. + */ +export async function grabLogsKeepDays() { + return readStepLimit({ + saved: await grabAppSetting({ key: LogsKeepDaysSettingKey }), + fallback: AppData.LogsKeepDays, + max: AppData.LogsKeepDaysMax, + }); +} + +/** + * The number with its default and ceiling, for the settings page. + */ +export default async function grabLogRetentionOverview() { + const overview: LogRetentionOverview = { + keep_days: await grabLogsKeepDays(), + default_days: AppData.LogsKeepDays, + max: AppData.LogsKeepDaysMax, + }; + + return overview; +} diff --git a/src/pages/admin/settings/(functions)/grab-settings-server-props.ts b/src/pages/admin/settings/(functions)/grab-settings-server-props.ts index 19b4895..b9fbca1 100644 --- a/src/pages/admin/settings/(functions)/grab-settings-server-props.ts +++ b/src/pages/admin/settings/(functions)/grab-settings-server-props.ts @@ -8,6 +8,7 @@ import type { BunextPageServerFn } from "@moduletrace/bunext/types"; import grabRunLimitsOverview from "@/src/functions/backend/db/app-settings/grab-run-limits"; import grabSkillLimitsOverview from "@/src/functions/backend/db/app-settings/grab-skill-limits"; import grabHookCallRetentionOverview from "@/src/functions/backend/db/app-settings/grab-hook-events-keep-days"; +import grabLogRetentionOverview from "@/src/functions/backend/db/app-settings/grab-logs-keep-days"; import grabChatroomRetentionOverview from "@/src/functions/backend/db/app-settings/grab-chatroom-retention"; import grabSessionLengthOverview from "@/src/functions/backend/db/app-settings/grab-bot-session-max-messages"; import grabStorageQuotaOverview from "@/src/functions/backend/db/app-settings/grab-storage-quota-overview"; @@ -53,6 +54,7 @@ export default async function grabSettingsServerProps({ req }: Params) { run_limits: await grabRunLimitsOverview().catch(() => null), skill_limits: await grabSkillLimitsOverview().catch(() => null), hook_call_retention: await grabHookCallRetentionOverview().catch(() => null), + log_retention: await grabLogRetentionOverview().catch(() => null), chatroom_retention: await grabChatroomRetentionOverview().catch(() => null), session_length: await grabSessionLengthOverview().catch(() => null), running_version: runningVersion({ env: process.env }), diff --git a/src/pages/admin/settings/general/(partials)/log-retention-form.tsx b/src/pages/admin/settings/general/(partials)/log-retention-form.tsx new file mode 100644 index 0000000..d3ab8ee --- /dev/null +++ b/src/pages/admin/settings/general/(partials)/log-retention-form.tsx @@ -0,0 +1,109 @@ +import { useState } from "react"; +import fetchApi from "@moduletrace/twui/utils/fetch/fetchApi"; +import type { APIResponseObject } from "@moduletrace/bunext/types"; +import Form from "@moduletrace/twui/form/Form"; +import Input from "@moduletrace/twui/form/Input"; +import type { LogRetentionOverview } from "@/src/types"; +import SettingsSaveBar from "../../(partials)/settings-save-bar"; + +type Props = { + retention?: LogRetentionOverview | null; +}; + +/** + * How many days logs are kept, saved on its own without a reload. + * Zero puts the code's default back. + */ +export default function LogRetentionForm({ retention }: Props) { + const [overview, setOverview] = useState(retention || undefined); + const [dirty, setDirty] = useState(false); + const [keep_days, setKeepDays] = useState(String(retention?.keep_days ?? "")); + /** + * The input only reads `defaultValue` when it mounts, so discarding or + * saving remounts it with the saved number. + */ + const [form_key, setFormKey] = useState(0); + const [loading, setLoading] = useState(false); + const [status, setStatus] = useState<{ + msg?: string; + error?: boolean; + success?: boolean; + }>(); + + const save = async () => { + if (!window.confirm("Save how long logs are kept? Entries older than that are deleted at the next daily prune.")) return; + + setLoading(true); + setStatus(undefined); + + const res = await fetchApi<{}, APIResponseObject>( + "/api/admin/settings/log-retention", + { + method: "POST", + body: { keep_days: Number(keep_days) || 0 }, + }, + ).catch(() => null); + + setLoading(false); + + if (!res?.success || !res.singleRes) { + setStatus({ + error: true, + msg: res?.msg || `Couldn't save it.`, + }); + return; + } + + setOverview(res.singleRes); + setKeepDays(String(res.singleRes.keep_days)); + setFormKey((key) => key + 1); + setDirty(false); + setStatus({ + success: true, + msg: `Saved: logs are kept for ${res.singleRes.keep_days} days.`, + }); + }; + + const discard = () => { + setKeepDays(String(overview?.keep_days ?? "")); + setFormKey((key) => key + 1); + setDirty(false); + setStatus(undefined); + }; + + return ( +
{ + event.preventDefault(); + save().catch(() => {}); + }} + className="w-full gap-3! items-stretch" + > + { + setKeepDays(event.target.value); + setDirty(true); + setStatus(undefined); + }} + suffix={`days`} + info={`Every cron run writes an entry, so a long time grows the database. 0 uses the default (${overview?.default_days ?? ""}).`} + /> + + + ); +} diff --git a/src/pages/admin/settings/general/(sections)/log-retention-section.tsx b/src/pages/admin/settings/general/(sections)/log-retention-section.tsx new file mode 100644 index 0000000..6e31545 --- /dev/null +++ b/src/pages/admin/settings/general/(sections)/log-retention-section.tsx @@ -0,0 +1,25 @@ +import { useContext } from "react"; +import { AppContext } from "@/src/pages/__root"; +import isPassThroughUser from "@/src/utils/rbac/is-pass-through-user"; +import SettingsGroup from "../../(partials)/settings-group"; +import LogRetentionForm from "../(partials)/log-retention-form"; + +/** + * How long logs (Activity) are kept. Only the super admin sees it, since + * only the super admin reads them. + */ +export default function LogRetentionSection() { + const { pageProps } = useContext(AppContext); + + if (!isPassThroughUser({ user_types: pageProps?.user_types })) return null; + + return ( + + + + ); +} diff --git a/src/pages/admin/settings/general/index.tsx b/src/pages/admin/settings/general/index.tsx index cfbdea4..050fcc1 100644 --- a/src/pages/admin/settings/general/index.tsx +++ b/src/pages/admin/settings/general/index.tsx @@ -3,6 +3,7 @@ import ChatroomRetentionSection from "./(sections)/chatroom-retention-section"; import HookCallRetentionSection from "./(sections)/hook-call-retention-section"; import InstallIconSection from "./(sections)/install-icon-section"; import InstallNameSection from "./(sections)/install-name-section"; +import LogRetentionSection from "./(sections)/log-retention-section"; import NotificationsSection from "./(sections)/notifications-section"; import PromptSection from "./(sections)/prompt-section"; import RunLimitsSection from "./(sections)/run-limits-section"; @@ -29,6 +30,7 @@ export default function AdminSettingsGeneralPage() { + diff --git a/src/pages/api/admin/settings/log-retention.ts b/src/pages/api/admin/settings/log-retention.ts new file mode 100644 index 0000000..4496406 --- /dev/null +++ b/src/pages/api/admin/settings/log-retention.ts @@ -0,0 +1,45 @@ +import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user"; +import grabLogRetentionOverview from "@/src/functions/backend/db/app-settings/grab-logs-keep-days"; +import saveAppSetting from "@/src/functions/backend/db/app-settings/save-app-setting"; +import logEvent, { grabRequestIP } from "@/src/functions/backend/logger"; +import { LogsKeepDaysSettingKey } from "@/src/dict/log-categories"; +import { SaveLogRetentionRequestSchema } from "@/src/schemas/log-schema"; +import type { LogRetentionOverview } from "@/src/types"; +import AppError from "@/src/utils/app-error"; +import parseWithSchema from "@/src/utils/parse-with-schema"; +import safeErrorMessage from "@/src/utils/safe-error-message"; +import type { APIResponseObject, BunextAPIRouteHandler } from "@moduletrace/bunext/types"; + +/** + * Super admin only: how many days logs are kept before the daily prune + * deletes them. Zero puts the code's default back. + */ +export const handler: BunextAPIRouteHandler> = async ({ req, body }) => { + try { + if (req.method != "POST") throw new AppError(`Method not allowed`); + + const user = await grabSuperAdminUser({ req }); + + if (!user?.id) throw new AppError(`Only a super admin can change how long logs are kept.`); + + const { keep_days } = parseWithSchema({ schema: SaveLogRetentionRequestSchema, data: body }); + + await saveAppSetting({ + key: LogsKeepDaysSettingKey, + value: keep_days > 0 ? String(keep_days) : null, + }); + + await logEvent({ + title: "Log retention changed", + category: "system", + user_id: user.id, + plain_text: `User #${user.id} set logs to be kept for ${keep_days || "the default number of"} days.`, + metadata: { keep_days }, + ip_address: grabRequestIP(req), + }); + + return { success: true, singleRes: await grabLogRetentionOverview() }; + } catch (error: unknown) { + return { success: false, msg: safeErrorMessage({ error }) }; + } +}; diff --git a/src/schemas/log-schema.ts b/src/schemas/log-schema.ts index 049419d..0dd1c6f 100644 --- a/src/schemas/log-schema.ts +++ b/src/schemas/log-schema.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { AppData } from "@/src/data/app-data"; import { LogCategories } from "@/src/dict/log-categories"; const LogCategoryValues = LogCategories.map((lc) => lc.value) as [ @@ -15,3 +16,11 @@ export const ClearLogsRequestSchema = z.object({ }); export type ClearLogsRequest = z.infer; + +/** + * How many days logs are kept (Settings > General). Zero puts the code's + * default back. + */ +export const SaveLogRetentionRequestSchema = z.object({ + keep_days: z.coerce.number().int().min(0).max(AppData.LogsKeepDaysMax), +}); diff --git a/src/types/index.ts b/src/types/index.ts index 086ea1c..32e8119 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1117,6 +1117,15 @@ export type HookCallRetentionOverview = { max: number; }; +/** + * How long logs are kept (Settings > General, `grab-logs-keep-days.ts`). + */ +export type LogRetentionOverview = { + keep_days: number; + default_days: number; + max: number; +}; + /** * The install's skill limits (Settings > General, `grab-skill-limits.ts`). */ @@ -1911,6 +1920,11 @@ export type PagePropsType = { * General settings page). */ hook_call_retention?: HookCallRetentionOverview | null; + /** + * How many days logs are kept, with the code's default (the General + * settings page). + */ + log_retention?: LogRetentionOverview | null; /** * How long closed chatrooms are kept and how many one place keeps, with * the code's defaults (the General settings page).