Archive the whole data volume in Backups, not just the database

Settings > Backups gains a whole-data archive beside the database
snapshots, and both lists now fully work from the page.

- Data archives: `write-data-backup-archive.ts` tars all of DATA_DIR into
  BACKUP_ARCHIVE_DIR as cobble-data-<epoch-ms>.tar.gz, every entry under one
  root folder named after the archive, writing a .part name and renaming on
  success so a cut-off run is never listed. The newest
  AppData.DataBackupsKept are kept; db/.backups is left as it is.
- Restore: `restore-data-backup.ts` extracts to a temp dir, requires one
  root folder holding a db folder, takes a safety archive first, then
  replaces DATA_DIR (the database snapshots included) and `restart-app.ts`
  restarts the process through the docker socket, since the open SQLite
  handle points at the replaced file.
- Upload and restore: a modal over each list (database file or data
  archive), replacing the old /admin/settings/backups/restore page.
- Delete for both kinds, icon-only row actions shared through
  RowIconButton, and restore on every row.
- One backup or restore of either kind at a time (with-backup-run-lock).
- BACKUP_ARCHIVE_DIR is a start-up path (ProcessOnlyEnvKeys); the installer
  mounts $COBBLE_HOME/backups-archives at /backups-archives and recreates a
  container that lacks it. The image installs tar and gzip.

src/pages/admin/settings/backups/(data)/data-backups-table-columns.ts

@@ -0,0 +1,12 @@
+export type DataBackupsTableColumn = {
+    title: string;
+    className?: string;
+    hide_title?: boolean;
+};
+
+export const DataBackupsTableColumns: DataBackupsTableColumn[] = [
+    { title: "Archive" },
+    { title: "Created" },
+    { title: "Size", className: "hidden md:table-cell" },
+    { title: "Actions", className: "text-right", hide_title: true },
+];

src/pages/admin/settings/backups/(functions)/create-data-backup.ts

@@ -0,0 +1,21 @@
+import adminFetchHandler from "@/src/functions/frontend/admin-fetch-handler";
+
+export default async function createDataBackup() {
+    if (
+        !window.confirm(
+            `Archive all of the data volume now? It can take a while on a busy install.`,
+        )
+    ) {
+        return { success: false, cancelled: true };
+    }
+
+    const res = await adminFetchHandler({
+        url: "/api/admin/backups/create-data",
+        method: "POST",
+    });
+
+    return {
+        success: Boolean(res.success),
+        msg: res.success ? undefined : res.msg || `Couldn't create archive`,
+    };
+}

src/pages/admin/settings/backups/(functions)/delete-data-backup.ts

@@ -0,0 +1,28 @@
+import adminFetchHandler from "@/src/functions/frontend/admin-fetch-handler";
+import type { DataBackup } from "@/src/types";
+import dayjs from "dayjs";
+
+type Params = {
+    backup: DataBackup;
+};
+
+export default async function deleteDataBackup({ backup }: Params) {
+    const confirm_msg = `Delete the data archive of ${dayjs(backup.created_at).format("MMM D, YYYY HH:mm:ss")}?\n\nThis removes the file for good.`;
+
+    if (!window.confirm(confirm_msg)) {
+        return { success: false, cancelled: true };
+    }
+
+    const res = await adminFetchHandler({
+        url: "/api/admin/backups/delete-data",
+        method: "POST",
+        body: {
+            backup_name: backup.name,
+        },
+    });
+
+    return {
+        success: Boolean(res.success),
+        msg: res.success ? undefined : res.msg || `Couldn't delete archive`,
+    };
+}

src/pages/admin/settings/backups/(functions)/delete-db-backup.ts

@@ -0,0 +1,28 @@
+import adminFetchHandler from "@/src/functions/frontend/admin-fetch-handler";
+import type { DbBackup } from "@/src/types";
+import dayjs from "dayjs";
+
+type Params = {
+    backup: DbBackup;
+};
+
+export default async function deleteDbBackup({ backup }: Params) {
+    const confirm_msg = `Delete the database snapshot of ${dayjs(backup.created_at).format("MMM D, YYYY HH:mm:ss")}?\n\nThis removes the file for good.`;
+
+    if (!window.confirm(confirm_msg)) {
+        return { success: false, cancelled: true };
+    }
+
+    const res = await adminFetchHandler({
+        url: "/api/admin/backups/delete-db",
+        method: "POST",
+        body: {
+            backup_name: backup.name,
+        },
+    });
+
+    return {
+        success: Boolean(res.success),
+        msg: res.success ? undefined : res.msg || `Couldn't delete snapshot`,
+    };
+}

src/pages/admin/settings/backups/(functions)/grab-backups-server-props.ts

@@ -1,5 +1,6 @@
 import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
 import grabDbBackups from "@/src/functions/backend/backups/grab-db-backups";
+import grabDataBackups from "@/src/functions/backend/backups/grab-data-backups";
 import { SettingsDefaultPath } from "@/src/dict/settings-pages-dict";
 import type { PagePropsType } from "@/src/types";
 import type { BunextPageServerFn } from "@moduletrace/bunext/types";
@@ -9,8 +10,8 @@ type Params = Parameters<BunextPageServerFn<PagePropsType>>[0];
 
 /**
  * Shared server props for the `/admin/settings/backups` sub-pages: the
- * saved snapshots. Super admins only; everyone else lands on the general
- * settings.
+ * saved database snapshots and the whole-data archives. Super admins
+ * only; everyone else lands on the general settings.
  */
 export default async function grabBackupsServerProps(params: Params) {
     const settings = await grabSettingsServerProps(params);
@@ -30,6 +31,7 @@ export default async function grabBackupsServerProps(params: Params) {
         props: {
             ...settings.props,
             backups: grabDbBackups(),
+            data_backups: grabDataBackups(),
         },
     };
 }

src/pages/admin/settings/backups/(functions)/restore-data-backup.ts

@@ -0,0 +1,33 @@
+import adminFetchHandler from "@/src/functions/frontend/admin-fetch-handler";
+import type { DataBackup } from "@/src/types";
+import dayjs from "dayjs";
+
+type Params = {
+    backup: DataBackup;
+};
+
+/**
+ * Replaces the whole data volume with the archive. The server takes a
+ * safety archive first and restarts afterwards, so this only sends the
+ * request and leaves the reload to the button.
+ */
+export default async function restoreDataBackup({ backup }: Params) {
+    const confirm_msg = `Restore the data volume from the archive of ${dayjs(backup.created_at).format("MMM D, YYYY HH:mm:ss")}?\n\nEvery bot, workspace, computer home, file, media item and database snapshot is replaced by the archive's, and the app restarts. A safety archive of the current data is made first.`;
+
+    if (!window.confirm(confirm_msg)) {
+        return { success: false, cancelled: true };
+    }
+
+    const res = await adminFetchHandler({
+        url: "/api/admin/backups/restore-data",
+        method: "POST",
+        body: {
+            backup_name: backup.name,
+        },
+    });
+
+    return {
+        success: Boolean(res.success),
+        msg: res.success ? undefined : res.msg || `Couldn't restore archive`,
+    };
+}

src/pages/admin/settings/backups/(functions)/upload-restore-backup.ts

@@ -4,7 +4,8 @@ type Params = {
 
 /**
  * Sends the file as multipart form data (not base64 JSON) so large dumps
- * aren't inflated.
+ * aren't inflated. The server saves it into the snapshots list, then
+ * restores it.
  */
 export default async function uploadRestoreBackup({ file }: Params) {
     const confirm_msg = `Restore the database from "${file.name}"?\n\nAll current data will be replaced with the data in this file. This cannot be undone. Consider creating a fresh backup first.`;

src/pages/admin/settings/backups/(functions)/upload-restore-data-backup.ts

@@ -0,0 +1,38 @@
+type Params = {
+    file: File;
+};
+
+/**
+ * Sends the file as multipart form data (not base64 JSON) so large archives
+ * aren't inflated. The app replaces the data volume and restarts, so the
+ * caller reloads the page once it is back.
+ */
+export default async function uploadRestoreDataBackup({ file }: Params) {
+    const confirm_msg = `Restore the data volume from "${file.name}"?\n\nEvery bot, workspace, computer home, file, media item and database snapshot is replaced by the archive's, and the app restarts. A safety archive of the current data is made first.`;
+
+    if (!window.confirm(confirm_msg)) {
+        return { success: false, cancelled: true };
+    }
+
+    try {
+        const form_data = new FormData();
+        form_data.append("file", file);
+
+        const res = await fetch("/api/admin/backups/upload-restore-data", {
+            method: "POST",
+            body: form_data,
+        });
+
+        const json = await res.json();
+
+        return {
+            success: Boolean(json.success),
+            msg: json.success ? undefined : json.msg || `Couldn't restore archive`,
+        };
+    } catch (error: any) {
+        return {
+            success: false,
+            msg: error.message || `Couldn't restore archive`,
+        };
+    }
+}

src/pages/admin/settings/backups/(partials)/backup-list-actions.tsx

@@ -1,12 +1,19 @@
-import { DatabaseBackup } from "lucide-react";
+import { DatabaseBackup, Upload } from "lucide-react";
 import Button from "@moduletrace/twui/layout/Button";
 import Tag from "@moduletrace/twui/elements/Tag";
 import Row from "@moduletrace/twui/layout/Row";
 import useStatus from "@moduletrace/twui/hooks/useStatus";
 import createBackup from "../(functions)/create-backup";
+import UploadRestoreModal from "../(partials)/upload-restore-modal";
+import { useState } from "react";
 
-export default function CreateBackupButton() {
+/**
+ * The database snapshot list's two ways in: make a fresh snapshot, or
+ * restore one from a file.
+ */
+export default function BackupListActions() {
     const { loading, setLoading, status, setStatus } = useStatus();
+    const [uploading, setUploading] = useState(false);
 
     const create_backup = async () => {
         setStatus(undefined);
@@ -29,6 +36,17 @@ export default function CreateBackupButton() {
     return (
         <Row className="gap-2! flex-nowrap">
             {status?.error && <Tag color="error">{status.msg}</Tag>}
+            <Button
+                title="Restore from a file"
+                variant="outlined"
+                color="gray"
+                beforeIcon={<Upload />}
+                onClick={() => {
+                    setUploading(true);
+                }}
+            >
+                Restore from file
+            </Button>
             <Button
                 title="Create Backup"
                 beforeIcon={<DatabaseBackup />}
@@ -39,6 +57,10 @@ export default function CreateBackupButton() {
             >
                 Create backup
             </Button>
+            <UploadRestoreModal
+                open={uploading}
+                onClose={() => setUploading(false)}
+            />
         </Row>
     );
 }

src/pages/admin/settings/backups/(partials)/backup-table-row.tsx

@@ -2,16 +2,13 @@ import Span from "@moduletrace/twui/layout/Span";
 import formatBytes from "@/src/utils/format-bytes";
 import type { DbBackup } from "@/src/types";
 import dayjs from "dayjs";
-import type { BackupsTableMode } from "./backups-table";
-import DownloadBackupButton from "./download-backup-button";
-import RestoreBackupButton from "./restore-backup-button";
+import DbBackupRowActions from "./db-backup-row-actions";
 
 type Props = {
     backup: DbBackup;
-    mode: BackupsTableMode;
 };
 
-export default function BackupTableRow({ backup, mode }: Props) {
+export default function BackupTableRow({ backup }: Props) {
     return (
         <tr className="border-t border-slate-200 dark:border-white/10">
             <td className="px-4 py-2.5 max-w-0 w-full">
@@ -34,11 +31,7 @@ export default function BackupTableRow({ backup, mode }: Props) {
                 </Span>
             </td>
             <td className="px-4 py-2 text-right whitespace-nowrap">
-                {mode == "restore" ? (
-                    <RestoreBackupButton backup={backup} />
-                ) : (
-                    <DownloadBackupButton backup={backup} />
-                )}
+                <DbBackupRowActions backup={backup} />
             </td>
         </tr>
     );

src/pages/admin/settings/backups/(partials)/backups-table.tsx

@@ -4,14 +4,11 @@ import { BackupsTableColumns } from "../(data)/backups-table-columns";
 import BackupTableRow from "./backup-table-row";
 import BackupsTableHeadCell from "./backups-table-head-cell";
 
-export type BackupsTableMode = "view" | "restore";
-
 type Props = {
     backups: DbBackup[];
-    mode: BackupsTableMode;
 };
 
-export default function BackupsTable({ backups, mode }: Props) {
+export default function BackupsTable({ backups }: Props) {
     return (
         <Paper className="p-0 gap-0 overflow-x-auto">
             <table className="w-full text-sm">
@@ -24,11 +21,7 @@ export default function BackupsTable({ backups, mode }: Props) {
                 </thead>
                 <tbody>
                     {backups.map((backup) => (
-                        <BackupTableRow
-                            backup={backup}
-                            mode={mode}
-                            key={backup.name}
-                        />
+                        <BackupTableRow backup={backup} key={backup.name} />
                     ))}
                 </tbody>
             </table>

src/pages/admin/settings/backups/(partials)/data-backup-list-actions.tsx

@@ -0,0 +1,67 @@
+import { Archive, Upload } from "lucide-react";
+import Button from "@moduletrace/twui/layout/Button";
+import Tag from "@moduletrace/twui/elements/Tag";
+import Row from "@moduletrace/twui/layout/Row";
+import useStatus from "@moduletrace/twui/hooks/useStatus";
+import createDataBackup from "../(functions)/create-data-backup";
+import UploadRestoreModal from "../(partials)/upload-restore-modal";
+import { useState } from "react";
+
+/**
+ * The data archive list's two ways in: make a fresh archive, or restore
+ * one from a file.
+ */
+export default function DataBackupListActions() {
+    const { loading, setLoading, status, setStatus } = useStatus();
+    const [uploading, setUploading] = useState(false);
+
+    const create_backup = async () => {
+        setStatus(undefined);
+        setLoading(true);
+
+        const res = await createDataBackup();
+
+        if (res.success) {
+            window.location.reload();
+            return;
+        }
+
+        setLoading(false);
+
+        if (res.msg) {
+            setStatus({ error: true, msg: res.msg });
+        }
+    };
+
+    return (
+        <Row className="gap-2! flex-nowrap">
+            {status?.error && <Tag color="error">{status.msg}</Tag>}
+            <Button
+                title="Restore from an archive file"
+                variant="outlined"
+                color="gray"
+                beforeIcon={<Upload />}
+                onClick={() => {
+                    setUploading(true);
+                }}
+            >
+                Restore from file
+            </Button>
+            <Button
+                title="Create Data Archive"
+                beforeIcon={<Archive />}
+                loading={loading}
+                onClick={() => {
+                    create_backup();
+                }}
+            >
+                Create archive
+            </Button>
+            <UploadRestoreModal
+                open={uploading}
+                onClose={() => setUploading(false)}
+                is_data
+            />
+        </Row>
+    );
+}

src/pages/admin/settings/backups/(partials)/data-backup-row-actions.tsx

@@ -0,0 +1,114 @@
+import Tag from "@moduletrace/twui/elements/Tag";
+import Row from "@moduletrace/twui/layout/Row";
+import useStatus from "@moduletrace/twui/hooks/useStatus";
+import type { DataBackup } from "@/src/types";
+import restoreDataBackup from "../(functions)/restore-data-backup";
+import deleteDataBackup from "../(functions)/delete-data-backup";
+import RowIconButton from "@/src/components/general/row-icon-button";
+import { ArchiveRestore, Download, Trash2 } from "lucide-react";
+import { useEffect, useRef, useState } from "react";
+
+/**
+ * How long the app is given to come back after a restore restarted it
+ * before the page is reloaded anyway.
+ */
+const RELOAD_DELAY_MS = 8000;
+
+type Props = {
+    backup: DataBackup;
+};
+
+/**
+ * One data archive's icon actions: restore (replaces the whole data
+ * volume, then the app restarts), download or delete, with their own
+ * one-line status.
+ */
+export default function DataBackupRowActions({ backup }: Props) {
+    const { loading, setLoading, status, setStatus } = useStatus();
+    const [restarting, setRestarting] = useState(false);
+    const timeout = useRef<ReturnType<typeof setTimeout>>(undefined);
+
+    useEffect(() => {
+        return () => {
+            if (timeout.current) clearTimeout(timeout.current);
+        };
+    }, []);
+
+    const remove = async () => {
+        setStatus(undefined);
+        setLoading(true);
+
+        const res = await deleteDataBackup({ backup });
+
+        if (res.success) {
+            window.location.reload();
+            return;
+        }
+
+        setLoading(false);
+
+        if (res.msg) {
+            setStatus({ error: true, msg: res.msg });
+        }
+    };
+
+    const restore = async () => {
+        setStatus(undefined);
+        setLoading(true);
+
+        const res = await restoreDataBackup({ backup });
+
+        if (res.success) {
+            setRestarting(true);
+            timeout.current = setTimeout(() => {
+                window.location.reload();
+            }, RELOAD_DELAY_MS);
+            return;
+        }
+
+        setLoading(false);
+
+        if (res.msg) {
+            setStatus({ error: true, msg: res.msg });
+        }
+    };
+
+    return (
+        <Row className="gap-1! flex-nowrap justify-end">
+            {status?.error && <Tag color="error">{status.msg}</Tag>}
+            {restarting && (
+                <Tag color="warning">Restoring, the app restarts soon...</Tag>
+            )}
+            <RowIconButton
+                icon={ArchiveRestore}
+                title="Restore this archive"
+                loading={loading && !restarting}
+                disabled={restarting}
+                onClick={() => {
+                    restore();
+                }}
+            />
+            <RowIconButton
+                icon={Download}
+                title="Download this archive"
+                href={`/api/admin/backups/download-data?backup_name=${encodeURIComponent(backup.name)}`}
+                linkProps={{
+                    download: backup.name,
+                    onClick: (e) => {
+                        if (!window.confirm(`Download archive ${backup.name}?`)) {
+                            e.preventDefault();
+                        }
+                    },
+                }}
+            />
+            <RowIconButton
+                icon={Trash2}
+                title="Delete this archive"
+                disabled={loading || restarting}
+                onClick={() => {
+                    remove();
+                }}
+            />
+        </Row>
+    );
+}

src/pages/admin/settings/backups/(partials)/data-backup-table-row.tsx

@@ -0,0 +1,38 @@
+import Span from "@moduletrace/twui/layout/Span";
+import formatBytes from "@/src/utils/format-bytes";
+import type { DataBackup } from "@/src/types";
+import dayjs from "dayjs";
+import DataBackupRowActions from "./data-backup-row-actions";
+
+type Props = {
+    backup: DataBackup;
+};
+
+export default function DataBackupTableRow({ backup }: Props) {
+    return (
+        <tr className="border-t border-slate-200 dark:border-white/10">
+            <td className="px-4 py-2.5 max-w-0 w-full">
+                <Span
+                    size="small"
+                    className="block truncate font-mono"
+                    title={backup.name}
+                >
+                    {backup.name}
+                </Span>
+            </td>
+            <td className="px-4 py-2.5 whitespace-nowrap">
+                <Span size="small">
+                    {dayjs(backup.created_at).format("MMM D, YYYY HH:mm:ss")}
+                </Span>
+            </td>
+            <td className="hidden md:table-cell px-4 py-2.5 whitespace-nowrap">
+                <Span size="small" variant="faded">
+                    {formatBytes({ bytes: backup.size })}
+                </Span>
+            </td>
+            <td className="px-4 py-2 text-right whitespace-nowrap">
+                <DataBackupRowActions backup={backup} />
+            </td>
+        </tr>
+    );
+}

src/pages/admin/settings/backups/(partials)/data-backups-table-head-cell.tsx

@@ -0,0 +1,21 @@
+import { twMerge } from "tailwind-merge";
+import Span from "@moduletrace/twui/layout/Span";
+import type { DataBackupsTableColumn } from "../(data)/data-backups-table-columns";
+
+export default function DataBackupsTableHeadCell({
+    title,
+    className,
+    hide_title,
+}: DataBackupsTableColumn) {
+    return (
+        <th className={twMerge("px-4 py-2.5 text-left font-medium", className)}>
+            <Span
+                size="smaller"
+                variant="faded"
+                className={twMerge(hide_title && "sr-only")}
+            >
+                {title}
+            </Span>
+        </th>
+    );
+}

src/pages/admin/settings/backups/(partials)/data-backups-table.tsx

@@ -0,0 +1,33 @@
+import Paper from "@moduletrace/twui/elements/Paper";
+import type { DataBackup } from "@/src/types";
+import { DataBackupsTableColumns } from "../(data)/data-backups-table-columns";
+import DataBackupTableRow from "./data-backup-table-row";
+import DataBackupsTableHeadCell from "./data-backups-table-head-cell";
+
+type Props = {
+    backups: DataBackup[];
+};
+
+export default function DataBackupsTable({ backups }: Props) {
+    return (
+        <Paper className="p-0 gap-0 overflow-x-auto">
+            <table className="w-full text-sm">
+                <thead className="bg-slate-50 dark:bg-white/5">
+                    <tr>
+                        {DataBackupsTableColumns.map((column) => (
+                            <DataBackupsTableHeadCell
+                                {...column}
+                                key={column.title}
+                            />
+                        ))}
+                    </tr>
+                </thead>
+                <tbody>
+                    {backups.map((backup) => (
+                        <DataBackupTableRow backup={backup} key={backup.name} />
+                    ))}
+                </tbody>
+            </table>
+        </Paper>
+    );
+}

src/pages/admin/settings/backups/(partials)/db-backup-row-actions.tsx

@@ -0,0 +1,74 @@
+import Button from "@moduletrace/twui/layout/Button";
+import Tag from "@moduletrace/twui/elements/Tag";
+import Row from "@moduletrace/twui/layout/Row";
+import useStatus from "@moduletrace/twui/hooks/useStatus";
+import type { DbBackup } from "@/src/types";
+import restoreBackup from "../(functions)/restore-backup";
+import deleteDbBackup from "../(functions)/delete-db-backup";
+import RowIconButton from "@/src/components/general/row-icon-button";
+import { ArchiveRestore, Download, Trash2 } from "lucide-react";
+
+type Props = {
+    backup: DbBackup;
+};
+
+/**
+ * One database snapshot's icon actions: restore, download or delete, with
+ * their own one-line status.
+ */
+export default function DbBackupRowActions({ backup }: Props) {
+    const { loading, setLoading, status, setStatus } = useStatus();
+
+    const run = async (task: () => Promise<{ success: boolean; msg?: string }>) => {
+        setStatus(undefined);
+        setLoading(true);
+
+        const res = await task();
+
+        if (res.success) {
+            window.location.reload();
+            return;
+        }
+
+        setLoading(false);
+
+        if (res.msg) {
+            setStatus({ error: true, msg: res.msg });
+        }
+    };
+
+    return (
+        <Row className="gap-1! flex-nowrap justify-end">
+            {status?.error && <Tag color="error">{status.msg}</Tag>}
+            <RowIconButton
+                icon={ArchiveRestore}
+                title="Restore this snapshot"
+                disabled={loading}
+                onClick={() => {
+                    run(() => restoreBackup({ backup }));
+                }}
+            />
+            <RowIconButton
+                icon={Download}
+                title="Download this snapshot"
+                href={`/api/admin/backups/download?backup_name=${encodeURIComponent(backup.name)}`}
+                linkProps={{
+                    download: backup.name,
+                    onClick: (e) => {
+                        if (!window.confirm(`Download backup ${backup.name}?`)) {
+                            e.preventDefault();
+                        }
+                    },
+                }}
+            />
+            <RowIconButton
+                icon={Trash2}
+                title="Delete this snapshot"
+                disabled={loading}
+                onClick={() => {
+                    run(() => deleteDbBackup({ backup }));
+                }}
+            />
+        </Row>
+    );
+}

src/pages/admin/settings/backups/(partials)/download-backup-button.tsx

@@ -1,30 +0,0 @@
-import Button from "@moduletrace/twui/layout/Button";
-import type { DbBackup } from "@/src/types";
-import { Download } from "lucide-react";
-
-type Props = {
-    backup: DbBackup;
-};
-
-export default function DownloadBackupButton({ backup }: Props) {
-    return (
-        <Button
-            title="Download Backup"
-            variant="outlined"
-            color="gray"
-            size="small"
-            beforeIcon={<Download size={20} />}
-            href={`/api/admin/backups/download?backup_name=${encodeURIComponent(backup.name)}`}
-            linkProps={{
-                download: backup.name,
-                onClick: (e) => {
-                    if (!window.confirm(`Download backup ${backup.name}?`)) {
-                        e.preventDefault();
-                    }
-                },
-            }}
-        >
-            Download
-        </Button>
-    );
-}

src/pages/admin/settings/backups/(partials)/restore-backup-button.tsx

@@ -1,52 +0,0 @@
-import Button from "@moduletrace/twui/layout/Button";
-import Tag from "@moduletrace/twui/elements/Tag";
-import Row from "@moduletrace/twui/layout/Row";
-import useStatus from "@moduletrace/twui/hooks/useStatus";
-import type { DbBackup } from "@/src/types";
-import restoreBackup from "../(functions)/restore-backup";
-import { ArchiveRestore } from "lucide-react";
-
-type Props = {
-    backup: DbBackup;
-};
-
-export default function RestoreBackupButton({ backup }: Props) {
-    const { loading, setLoading, status, setStatus } = useStatus();
-
-    const restore_backup = async () => {
-        setStatus(undefined);
-        setLoading(true);
-
-        const res = await restoreBackup({ backup });
-
-        if (res.success) {
-            window.location.reload();
-            return;
-        }
-
-        setLoading(false);
-
-        if (res.msg) {
-            setStatus({ error: true, msg: res.msg });
-        }
-    };
-
-    return (
-        <Row className="gap-2! flex-nowrap justify-end">
-            {status?.error && <Tag color="error">{status.msg}</Tag>}
-            <Button
-                title="Restore Backup"
-                variant="outlined"
-                color="warning"
-                size="small"
-                beforeIcon={<ArchiveRestore size={20} />}
-                loading={loading}
-                onClick={() => {
-                    restore_backup();
-                }}
-            >
-                Restore
-            </Button>
-        </Row>
-    );
-}

src/pages/admin/settings/backups/(partials)/upload-restore-fields.tsx

@@ -1,4 +1,3 @@
-import { Upload } from "lucide-react";
 import FileUpload from "@moduletrace/twui/form/FileUpload";
 import Button from "@moduletrace/twui/layout/Button";
 import Tag from "@moduletrace/twui/elements/Tag";
@@ -6,22 +5,56 @@ import Row from "@moduletrace/twui/layout/Row";
 import Stack from "@moduletrace/twui/layout/Stack";
 import useStatus from "@moduletrace/twui/hooks/useStatus";
 import { useState } from "react";
-import uploadRestoreBackup from "../(functions)/upload-restore-backup";
+import { Upload } from "lucide-react";
 
-export default function UploadRestoreForm() {
+type Props = {
+    /**
+     * The file picker's hint, e.g. "Click or drop a SQLite database file".
+     */
+    label: string;
+    /**
+     * What the picker accepts, e.g. `.db,.sqlite,.sqlite3`.
+     */
+    accept?: string;
+    /**
+     * The button's label.
+     */
+    action_label: string;
+    /**
+     * Sends the file and reports the outcome.
+     */
+    upload: (params: { file: File }) => Promise<{ success: boolean; msg?: string }>;
+    /**
+     * Reload the page once it worked (the app restarts after an archive
+     * restore, so the caller waits for it first).
+     */
+    onDone: () => void;
+};
+
+/**
+ * Picks one file and sends it for a restore: shared by the database upload
+ * and the data archive upload, which differ only in what they accept.
+ */
+export default function UploadRestoreFields({
+    label,
+    accept,
+    action_label,
+    upload,
+    onDone,
+}: Props) {
     const { loading, setLoading, status, setStatus } = useStatus();
     const [file, setFile] = useState<File>();
 
-    const upload_restore = async () => {
+    const send = async () => {
         if (!file) return;
 
         setStatus(undefined);
         setLoading(true);
 
-        const res = await uploadRestoreBackup({ file });
+        const res = await upload({ file });
 
         if (res.success) {
-            window.location.reload();
+            onDone();
             return;
         }
 
@@ -35,9 +68,9 @@ export default function UploadRestoreForm() {
     return (
         <Stack className="w-full items-stretch gap-4">
             <FileUpload
-                label="Click or drop a SQLite database file"
+                label={label}
                 className="h-[220px]"
-                fileInputProps={{ accept: ".db,.sqlite,.sqlite3" }}
+                fileInputProps={accept ? { accept } : undefined}
                 changeHandler={(file_data) => {
                     setStatus(undefined);
                     setFile(file_data?.file);
@@ -46,16 +79,16 @@ export default function UploadRestoreForm() {
             <Row className="w-full justify-end gap-2! flex-nowrap">
                 {status?.error && <Tag color="error">{status.msg}</Tag>}
                 <Button
-                    title="Upload And Restore Backup"
+                    title={action_label}
                     color="warning"
                     beforeIcon={<Upload />}
                     loading={loading}
                     disabled={!file || loading}
                     onClick={() => {
-                        upload_restore();
+                        send();
                     }}
                 >
-                    Upload & restore
+                    {action_label}
                 </Button>
             </Row>
         </Stack>

src/pages/admin/settings/backups/(partials)/upload-restore-modal.tsx

@@ -0,0 +1,72 @@
+import AddModalShell from "@/src/layouts/bots/(partials)/add-modal-shell";
+import UploadRestoreFields from "./upload-restore-fields";
+import uploadRestoreBackup from "../(functions)/upload-restore-backup";
+import uploadRestoreDataBackup from "../(functions)/upload-restore-data-backup";
+
+/**
+ * How long the app is given to come back after an archive restore
+ * restarted it before the page is reloaded anyway.
+ */
+const RELOAD_DELAY_MS = 8000;
+
+type Props = {
+    open: boolean;
+    onClose: () => void;
+    /**
+     * True for an upload listing a data archive, which restarts the app.
+     */
+    is_data?: boolean;
+};
+
+/**
+ * Uploading a file and restoring it, in a modal over the list it came
+ * from. A database file is restored in place; a data archive replaces the
+ * whole data volume, so the page waits for the app to restart.
+ */
+export default function UploadRestoreModal({ open, onClose, is_data }: Props) {
+    const reload = () => {
+        if (!is_data) {
+            window.location.reload();
+            return;
+        }
+
+        setTimeout(() => {
+            window.location.reload();
+        }, RELOAD_DELAY_MS);
+    };
+
+    return (
+        <AddModalShell
+            open={open}
+            setOpen={(next) => !next && onClose()}
+            title={is_data ? "Restore from an archive" : "Restore from a file"}
+            description={
+                is_data
+                    ? "Upload a whole-data tar.gz archive. It replaces every folder and the database snapshots, a safety archive is made first, and the app restarts."
+                    : "Upload a SQLite database file. It is saved to the snapshots list, then restored."
+            }
+        >
+            {open ? (
+                <UploadRestoreFields
+                    label={
+                        is_data
+                            ? "Click or drop a data archive"
+                            : "Click or drop a SQLite database file"
+                    }
+                    accept={
+                        is_data
+                            ? ".tar.gz,.tgz,application/gzip"
+                            : ".db,.sqlite,.sqlite3"
+                    }
+                    action_label={is_data ? "Upload & restore archive" : "Upload & restore"}
+                    upload={
+                        is_data
+                            ? uploadRestoreDataBackup
+                            : uploadRestoreBackup
+                    }
+                    onDone={reload}
+                />
+            ) : null}
+        </AddModalShell>
+    );
+}

src/pages/admin/settings/backups/(sections)/backups-list-section.tsx

@@ -3,39 +3,31 @@ import Row from "@moduletrace/twui/layout/Row";
 import Span from "@moduletrace/twui/layout/Span";
 import SettingsGroup from "../../(partials)/settings-group";
 import { AppContext } from "@/src/pages/__root";
-import CreateBackupButton from "../(partials)/create-backup-button";
-import BackupsTable, { type BackupsTableMode } from "../(partials)/backups-table";
-
-type Props = {
-    mode: BackupsTableMode;
-};
+import BackupListActions from "../(partials)/backup-list-actions";
+import BackupsTable from "../(partials)/backups-table";
 
 /**
- * The database snapshots, newest first. On the restore page each row
- * restores; otherwise each one downloads, and a new backup can be made.
+ * The database snapshots, newest first. Each row restores, downloads or
+ * deletes, and a fresh one can be made or uploaded. These are the
+ * bun-sqlite snapshots in `db/.backups`, not the whole-data archives.
  */
-export default function BackupsListSection({ mode }: Props) {
+export default function BackupsListSection() {
     const { pageProps } = useContext(AppContext);
     const backups = pageProps?.backups || [];
-    const is_restore = mode == "restore";
 
     return (
         <SettingsGroup
-            title={is_restore ? "Saved snapshots" : "Snapshots"}
-            description={
-                is_restore
-                    ? "Restore any of these, or upload a file below. Create a fresh backup first if you might need the current data."
-                    : "Older snapshots are pruned automatically. Download one to keep it."
-            }
+            title="Database snapshots"
+            description="Consistent snapshots of the database only, kept beside it. Older ones are pruned automatically. Download one to keep it."
         >
             <Row className="w-full items-center justify-between gap-2! flex-wrap">
                 <Span size="small" variant="faded">
                     {`${backups.length} backup${backups.length == 1 ? "" : "s"}`}
                 </Span>
-                {is_restore ? null : <CreateBackupButton />}
+                <BackupListActions />
             </Row>
             {backups[0] ? (
-                <BackupsTable backups={backups} mode={mode} />
+                <BackupsTable backups={backups} />
             ) : (
                 <Span size="small" variant="faded">
                     No backups yet.

src/pages/admin/settings/backups/(sections)/data-backups-section.tsx

@@ -0,0 +1,39 @@
+import { useContext } from "react";
+import Row from "@moduletrace/twui/layout/Row";
+import Span from "@moduletrace/twui/layout/Span";
+import SettingsGroup from "../../(partials)/settings-group";
+import { AppContext } from "@/src/pages/__root";
+import DataBackupListActions from "../(partials)/data-backup-list-actions";
+import DataBackupsTable from "../(partials)/data-backups-table";
+
+/**
+ * The tar.gz archives of the whole data volume, newest first. Each one
+ * restores (replacing every folder and the database snapshots with the
+ * archive's own, then the app restarts), downloads or deletes; the newest
+ * `AppData.DataBackupsKept` are kept and older ones pruned.
+ */
+export default function DataBackupsSection() {
+    const { pageProps } = useContext(AppContext);
+    const backups = pageProps?.data_backups || [];
+
+    return (
+        <SettingsGroup
+            title="Data archives"
+            description="One tar.gz of the whole data volume, under a single folder named after the archive: the database, media, computer homes, every bot and workspace file, and the database snapshots. They live outside the data volume, and the newest few are kept."
+        >
+            <Row className="w-full items-center justify-between gap-2! flex-wrap">
+                <Span size="small" variant="faded">
+                    {`${backups.length} archive${backups.length == 1 ? "" : "s"}`}
+                </Span>
+                <DataBackupListActions />
+            </Row>
+            {backups[0] ? (
+                <DataBackupsTable backups={backups} />
+            ) : (
+                <Span size="small" variant="faded">
+                    No archives yet.
+                </Span>
+            )}
+        </SettingsGroup>
+    );
+}

src/pages/admin/settings/backups/(sections)/restore-backups-section.tsx

@@ -1,28 +0,0 @@
-import Button from "@moduletrace/twui/layout/Button";
-import { ArchiveRestore } from "lucide-react";
-import SettingsGroup from "../../(partials)/settings-group";
-import { SettingsBackupsRestorePath } from "@/src/dict/settings-pages-dict";
-
-/**
- * The way into the restore page from the backups list.
- */
-export default function RestoreBackupsSection() {
-    return (
-        <SettingsGroup
-            title="Restore"
-            description="Replace all current data with the contents of a saved or uploaded backup. This cannot be undone."
-        >
-            <Button
-                title="Restore backup"
-                href={SettingsBackupsRestorePath}
-                variant="outlined"
-                color="warning"
-                size="small"
-                beforeIcon={<ArchiveRestore size={20} />}
-                className="self-start"
-            >
-                Restore a backup
-            </Button>
-        </SettingsGroup>
-    );
-}

src/pages/admin/settings/backups/index.tsx

@@ -1,5 +1,5 @@
 import BackupsListSection from "./(sections)/backups-list-section";
-import RestoreBackupsSection from "./(sections)/restore-backups-section";
+import DataBackupsSection from "./(sections)/data-backups-section";
 import SettingsShell from "../(partials)/settings-shell";
 import settingsPageMeta from "../(functions)/settings-page-meta";
 
@@ -7,11 +7,11 @@ export default function AdminSettingsBackupsPage() {
     return (
         <SettingsShell
             title="Backups"
-            description="Snapshots of the database, and restoring one."
+            description="Database snapshots and whole-data archives: make them, restore from one, and delete them."
             wide
         >
-            <BackupsListSection mode="view" />
-            <RestoreBackupsSection />
+            <BackupsListSection />
+            <DataBackupsSection />
         </SettingsShell>
     );
 }

src/pages/admin/settings/backups/restore/(sections)/upload-restore-section.tsx

@@ -1,16 +0,0 @@
-import SettingsGroup from "../../../(partials)/settings-group";
-import UploadRestoreForm from "../(partials)/upload-restore-form";
-
-/**
- * Restoring from a file on this machine, for a backup kept elsewhere.
- */
-export default function UploadRestoreSection() {
-    return (
-        <SettingsGroup
-            title="From a file"
-            description="Upload a SQLite database file. It is saved to the backups list, then restored."
-        >
-            <UploadRestoreForm />
-        </SettingsGroup>
-    );
-}

src/pages/admin/settings/backups/restore/index.server.ts

@@ -1,7 +0,0 @@
-import type { PagePropsType } from "@/src/types";
-import type { BunextPageServerFn } from "@moduletrace/bunext/types";
-import grabBackupsServerProps from "../(functions)/grab-backups-server-props";
-
-export const server: BunextPageServerFn<PagePropsType> = async (params) => {
-    return await grabBackupsServerProps(params);
-};

src/pages/admin/settings/backups/restore/index.tsx

@@ -1,26 +0,0 @@
-import UploadRestoreSection from "./(sections)/upload-restore-section";
-import BackupsListSection from "../(sections)/backups-list-section";
-import SettingsShell from "../../(partials)/settings-shell";
-import type { BunextPageModuleMetaFn } from "@moduletrace/bunext/types";
-import { SiteData } from "@/src/data/site-data";
-
-export default function AdminSettingsRestoreBackupPage() {
-    return (
-        <SettingsShell
-            title="Restore backup"
-            description="Replace all current data with a saved or uploaded backup."
-            breadcrumb_title="Restore backup"
-            wide
-        >
-            <UploadRestoreSection />
-            <BackupsListSection mode="restore" />
-        </SettingsShell>
-    );
-}
-
-export const meta: BunextPageModuleMetaFn = async () => {
-    return {
-        title: `Restore backup | ${SiteData["SiteName"]}`,
-        description: `Restore a database backup on ${SiteData["SiteName"]}.`,
-    };
-};

src/pages/api/admin/backups/create-data.ts

@@ -0,0 +1,52 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import createDataBackup from "@/src/functions/backend/backups/create-data-backup";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Archives the whole `DATA_DIR` (Settings > Backups > Data archives).
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    try {
+        const backup = await createDataBackup();
+
+        await logEvent({
+            title: "Data backup created",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} created a data backup (${backup.name}).`,
+            metadata: { backup_name: backup.name },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+        };
+    } catch (error: any) {
+        await logError({
+            title: "Data backup failed",
+            error,
+            user_id: user.id,
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};

src/pages/api/admin/backups/delete-data.ts

@@ -0,0 +1,57 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import deleteDataBackup from "@/src/functions/backend/backups/delete-data-backup";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import type { ApiReqParams } from "@/src/types";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Deletes one whole-data archive.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+    body,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    const backup_name = (body as ApiReqParams | undefined)?.backup_name;
+
+    try {
+        deleteDataBackup({ backup_name });
+
+        await logEvent({
+            title: "Data backup deleted",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} deleted data archive ${backup_name}.`,
+            metadata: { backup_name },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+        };
+    } catch (error: any) {
+        await logError({
+            title: "Data backup deletion failed",
+            error,
+            user_id: user.id,
+            metadata: { backup_name },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};

src/pages/api/admin/backups/delete-db.ts

@@ -0,0 +1,57 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import deleteDbBackup from "@/src/functions/backend/backups/delete-db-backup";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import type { ApiReqParams } from "@/src/types";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Deletes one database snapshot from `db/.backups`.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+    body,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    const backup_name = (body as ApiReqParams | undefined)?.backup_name;
+
+    try {
+        deleteDbBackup({ backup_name });
+
+        await logEvent({
+            title: "Database backup deleted",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} deleted database backup ${backup_name}.`,
+            metadata: { backup_name },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: true,
+        };
+    } catch (error: any) {
+        await logError({
+            title: "Database backup deletion failed",
+            error,
+            user_id: user.id,
+            metadata: { backup_name },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};

src/pages/api/admin/backups/download-data.ts

@@ -0,0 +1,43 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import resolveDataBackupPath from "@/src/functions/backend/backups/resolve-data-backup-path";
+import type { ApiReqParams } from "@/src/types";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+import path from "path";
+
+/**
+ * Streams a whole-data archive back, by name.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+    query,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    try {
+        const backup_path = resolveDataBackupPath({
+            backup_name: (query as ApiReqParams | undefined)?.backup_name,
+        });
+
+        return new Response(Bun.file(backup_path), {
+            headers: {
+                "Content-Type": "application/gzip",
+                "Content-Disposition": `attachment; filename="${path.basename(backup_path)}"`,
+            },
+        });
+    } catch (error: any) {
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};

src/pages/api/admin/backups/restore-data.ts

@@ -0,0 +1,66 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import restoreDataBackup from "@/src/functions/backend/backups/restore-data-backup";
+import restartApp from "@/src/functions/backend/backups/restart-app";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import type { ApiReqParams } from "@/src/types";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+} from "@moduletrace/bunext/types";
+
+/**
+ * Replaces the whole data volume with a saved archive, then restarts the
+ * app so it opens the restored database.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+    body,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    const backup_name = (body as ApiReqParams | undefined)?.backup_name;
+
+    try {
+        const result = await restoreDataBackup({ backup_name });
+
+        await logEvent({
+            title: "Data restored",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} restored the data volume from archive ${result.restored_name} (safety copy ${result.safety_name}). The app restarted.`,
+            metadata: {
+                backup_name: result.restored_name,
+                safety_name: result.safety_name,
+            },
+            ip_address: grabRequestIP(req),
+        });
+
+        restartApp();
+
+        return {
+            success: true,
+        };
+    } catch (error: any) {
+        await logError({
+            title: "Data restore failed",
+            error,
+            user_id: user.id,
+            metadata: {
+                backup_name,
+            },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};

src/pages/api/admin/backups/upload-restore-data.ts

@@ -0,0 +1,82 @@
+import grabSuperAdminUser from "@/src/functions/backend/backups/grab-super-admin-user";
+import restoreDataBackup from "@/src/functions/backend/backups/restore-data-backup";
+import restartApp from "@/src/functions/backend/backups/restart-app";
+import saveUploadedDataBackup from "@/src/functions/backend/backups/save-uploaded-data-backup";
+import logEvent, { grabRequestIP, logError } from "@/src/functions/backend/logger";
+import type {
+    APIResponseObject,
+    BunextAPIRouteHandler,
+    BunextServerRouteConfig,
+} from "@moduletrace/bunext/types";
+
+export const config: BunextServerRouteConfig = {
+    max_request_body_mb: 500,
+};
+
+/**
+ * Multipart upload (`file` field) of a whole-data archive. The file is
+ * saved into the archive directory, then restored, and the app restarts.
+ */
+export const handler: BunextAPIRouteHandler<APIResponseObject> = async ({
+    req,
+}) => {
+    const user = await grabSuperAdminUser({ req });
+
+    if (!user?.id) {
+        return {
+            success: false,
+            msg: `Unauthorized`,
+        };
+    }
+
+    let file_name: string | undefined;
+    let backup_name: string | undefined;
+
+    try {
+        const file = (await req.formData()).get("file");
+
+        if (!(file instanceof File)) {
+            throw new Error(`No archive uploaded`);
+        }
+
+        file_name = file.name;
+        backup_name = await saveUploadedDataBackup({ file });
+
+        const result = await restoreDataBackup({ backup_name });
+
+        await logEvent({
+            title: "Data restored from upload",
+            category: "system",
+            user_id: user.id,
+            plain_text: `User #${user.id} restored the data volume from uploaded archive ${file_name} (saved as ${result.restored_name}, safety copy ${result.safety_name}). The app restarted.`,
+            metadata: {
+                file_name,
+                backup_name: result.restored_name,
+                safety_name: result.safety_name,
+            },
+            ip_address: grabRequestIP(req),
+        });
+
+        restartApp();
+
+        return {
+            success: true,
+        };
+    } catch (error: any) {
+        await logError({
+            title: "Data restore from upload failed",
+            error,
+            user_id: user.id,
+            metadata: {
+                file_name,
+                backup_name,
+            },
+            ip_address: grabRequestIP(req),
+        });
+
+        return {
+            success: false,
+            msg: error.message,
+        };
+    }
+};