Run whole-data archives and restores in the background

A data archive or restore ran inside its request, which sent nothing
until tar finished: Bun closes a silent socket after 10s (its default
idleTimeout in production) and a proxy cuts it at about 100s, so the
page showed a network error while the archive kept going, and a retry
said another backup was already running.

The routes now start the run (startDataBackupRun, refused while the
backup lock is held) and answer at once. The page follows it by polling
api/admin/backups/data-status: an archive reloads once written, a
restore once the status comes from a new process after its restart,
and a failure shows its error. A run in progress survives a reload.

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/functions/backend/backups/data-backup-run.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, test } from "bun:test";
+import AppError from "@/src/utils/app-error";
+import { BootId, grabDataBackupRun, startDataBackupRun } from "./data-backup-run";
+import { isBackupRunning } from "./with-backup-run-lock";
+
+function deferred() {
+    let resolve!: (name: string) => void;
+    let reject!: (error: Error) => void;
+    const promise = new Promise<string>((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return { promise, resolve, reject };
+}
+
+const settle = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+describe("startDataBackupRun", () => {
+    test("answers at once, then records the finished archive", async () => {
+        const task = deferred();
+
+        const run = startDataBackupRun({
+            kind: "archive",
+            task: () => task.promise,
+        });
+
+        expect(run.status).toBe("running");
+        expect(run.kind).toBe("archive");
+        expect(run.boot_id).toBe(BootId);
+        expect(isBackupRunning()).toBe(true);
+
+        task.resolve("cobble-data-1.tar.gz");
+        await settle();
+
+        const done = grabDataBackupRun();
+        expect(done.status).toBe("done");
+        expect(done.name).toBe("cobble-data-1.tar.gz");
+        expect(isBackupRunning()).toBe(false);
+    });
+
+    test("refuses a second run while one is going, before it starts", async () => {
+        const task = deferred();
+        let second_started = false;
+
+        startDataBackupRun({ kind: "archive", task: () => task.promise });
+
+        expect(() =>
+            startDataBackupRun({
+                kind: "restore",
+                task: async () => {
+                    second_started = true;
+                    return "x";
+                },
+            }),
+        ).toThrow(AppError);
+        expect(second_started).toBe(false);
+
+        task.resolve("cobble-data-2.tar.gz");
+        await settle();
+    });
+
+    test("records a failure and frees the lock for the next run", async () => {
+        const task = deferred();
+
+        startDataBackupRun({ kind: "restore", task: () => task.promise });
+        task.reject(new Error("tar: broken"));
+        await settle();
+
+        const failed = grabDataBackupRun();
+        expect(failed.status).toBe("failed");
+        expect(failed.kind).toBe("restore");
+        expect(failed.error).toBe("tar: broken");
+        expect(isBackupRunning()).toBe(false);
+
+        const next = startDataBackupRun({
+            kind: "archive",
+            task: async () => "cobble-data-3.tar.gz",
+        });
+        expect(next.status).toBe("running");
+        await settle();
+        expect(grabDataBackupRun().status).toBe("done");
+    });
+});

src/functions/backend/backups/data-backup-run.ts

@@ -0,0 +1,66 @@
+import AppError from "@/src/utils/app-error";
+import type { DataBackupRun, DataBackupRunKind } from "@/src/types";
+import withBackupRunLock, { isBackupRunning } from "./with-backup-run-lock";
+
+/**
+ * Changes every time the process starts, so the page can tell a restore
+ * restarted the app (the run it started belonged to another boot).
+ */
+export const BootId = crypto.randomUUID();
+
+let run: Omit<DataBackupRun, "boot_id"> | null = null;
+
+type Params = {
+    kind: DataBackupRunKind;
+    /**
+     * The archive or restore itself. It resolves to the archive's name.
+     */
+    task: () => Promise<string>;
+};
+
+/**
+ * Starts a whole-data archive or restore in the background and hands back
+ * its state at once. Both take far longer than a request may stay silent
+ * (Bun closes an idle socket after 10s, a proxy after about 100s), so the
+ * route answers straight away and the page polls `data-status` until the
+ * run is done or failed. Throws while any backup or restore is running,
+ * before anything starts.
+ */
+export function startDataBackupRun({ kind, task }: Params): DataBackupRun {
+    if (isBackupRunning() || run?.status == "running") {
+        throw new AppError(`Another backup or restore is already running`);
+    }
+
+    const started = {
+        kind,
+        status: "running" as const,
+        started_at: Date.now(),
+    };
+
+    run = started;
+
+    withBackupRunLock({ task })
+        .then((name) => {
+            if (run !== started) return;
+            run = { ...started, status: "done", name, finished_at: Date.now() };
+        })
+        .catch((error: unknown) => {
+            if (run !== started) return;
+            run = {
+                ...started,
+                status: "failed",
+                error: error instanceof Error ? error.message : String(error),
+                finished_at: Date.now(),
+            };
+        });
+
+    return grabDataBackupRun();
+}
+
+/**
+ * The last whole-data run this process started, if any, with this boot's
+ * id.
+ */
+export function grabDataBackupRun(): DataBackupRun {
+    return { ...(run ?? { status: "idle" }), boot_id: BootId };
+}

src/functions/backend/backups/save-uploaded-data-backup.ts

@@ -2,6 +2,7 @@ import fs from "fs";
 import path from "path";
 import { AppData } from "@/src/data/app-data";
 import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
+import AppError from "@/src/utils/app-error";
 
 type Params = {
     file: File;
@@ -18,13 +19,13 @@ const GZIP_MAGIC = [0x1f, 0x8b];
  */
 export default async function saveUploadedDataBackup({ file }: Params) {
     if (!file.size) {
-        throw new Error(`The uploaded archive is empty`);
+        throw new AppError(`The uploaded archive is empty`);
     }
 
     const head = new Uint8Array(await file.slice(0, 2).arrayBuffer());
 
     if (head[0] != GZIP_MAGIC[0] || head[1] != GZIP_MAGIC[1]) {
-        throw new Error(`Only a gzipped tar archive can be restored`);
+        throw new AppError(`Only a gzipped tar archive can be restored`);
     }
 
     const archive_dir = grabBackupArchiveDir();

src/functions/backend/backups/with-backup-run-lock.ts

@@ -4,6 +4,14 @@ type Params<T> = {
     task: () => Promise<T>;
 };
 
+/**
+ * Whether a backup or restore of any kind holds the lock right now, so a
+ * background run can be refused before it starts.
+ */
+export function isBackupRunning() {
+    return is_running;
+}
+
 /**
  * Runs one backup or restore at a time, whichever kind it is, so a
  * database snapshot, a database restore and a whole-data archive never