Archive the whole data volume in Backups, not just the database
Settings > Backups gains a whole-data archive beside the database snapshots, and both lists now fully work from the page. - Data archives: `write-data-backup-archive.ts` tars all of DATA_DIR into BACKUP_ARCHIVE_DIR as cobble-data-<epoch-ms>.tar.gz, every entry under one root folder named after the archive, writing a .part name and renaming on success so a cut-off run is never listed. The newest AppData.DataBackupsKept are kept; db/.backups is left as it is. - Restore: `restore-data-backup.ts` extracts to a temp dir, requires one root folder holding a db folder, takes a safety archive first, then replaces DATA_DIR (the database snapshots included) and `restart-app.ts` restarts the process through the docker socket, since the open SQLite handle points at the replaced file. - Upload and restore: a modal over each list (database file or data archive), replacing the old /admin/settings/backups/restore page. - Delete for both kinds, icon-only row actions shared through RowIconButton, and restore on every row. - One backup or restore of either kind at a time (with-backup-run-lock). - BACKUP_ARCHIVE_DIR is a start-up path (ProcessOnlyEnvKeys); the installer mounts $COBBLE_HOME/backups-archives at /backups-archives and recreates a container that lacks it. The image installs tar and gzip.
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
import withBackupRunLock from "./with-backup-run-lock";
|
||||
import writeDataBackupArchive from "./write-data-backup-archive";
|
||||
|
||||
/**
|
||||
* Archives the whole `DATA_DIR` (database, media, computer homes, bot and
|
||||
* workspace files, MCP installs) into `BACKUP_ARCHIVE_DIR` as
|
||||
* `cobble-data-{timestamp}.tar.gz`, then keeps only the newest
|
||||
* `AppData.DataBackupsKept`. Every entry sits under one root folder named
|
||||
* after the archive (`cobble-data-{timestamp}/`), so extracting it lands
|
||||
* one folder, never a heap of files in the current directory. The
|
||||
* database's own snapshots in `db/.backups` are inside `DATA_DIR`, so they
|
||||
* ride along and are left alone on disk.
|
||||
*
|
||||
* The archive is a filesystem copy of a live volume, so the SQLite file
|
||||
* inside it is only as consistent as the moment tar read it; a consistent
|
||||
* database snapshot is what the `db/.backups` list is for. Restoring a
|
||||
* whole-data archive is `restoreDataBackup` (`restore-data-backup.ts`).
|
||||
*/
|
||||
export default async function createDataBackup() {
|
||||
return await withBackupRunLock({ task: writeDataBackupArchive });
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
|
||||
import resolveDataBackupPath from "./resolve-data-backup-path";
|
||||
|
||||
type Params = {
|
||||
backup_name?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Deletes one whole-data archive. Only an existing archive's own file is
|
||||
* removed, never a path from the request.
|
||||
*/
|
||||
export default function deleteDataBackup({ backup_name }: Params) {
|
||||
const backup_path = resolveDataBackupPath({ backup_name });
|
||||
|
||||
fs.rmSync(path.join(grabBackupArchiveDir(), path.basename(backup_path)), {
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import fs from "fs";
|
||||
import resolveDbBackupPath from "./resolve-db-backup-path";
|
||||
|
||||
type Params = {
|
||||
backup_name?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Deletes one database snapshot from `db/.backups`. Only an existing
|
||||
* snapshot's own file is removed, never a path from the request.
|
||||
*/
|
||||
export default function deleteDbBackup({ backup_name }: Params) {
|
||||
fs.rmSync(resolveDbBackupPath({ backup_name }), { force: true });
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import type { DataBackup } from "@/src/types";
|
||||
import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
|
||||
|
||||
const DATA_BACKUP_NAME_REGEX = new RegExp(
|
||||
`^${AppData.DataBackupNamePrefix}(\\d+)\\.tar\\.gz$`,
|
||||
);
|
||||
|
||||
/**
|
||||
* Lists the `cobble-data-{timestamp}.tar.gz` archives of the whole
|
||||
* `DATA_DIR`, newest first. Only finished files match: a run in progress
|
||||
* writes a `.part` name, so a half-written archive never shows up. The
|
||||
* database snapshots bun-sqlite writes into `db/.backups` never match
|
||||
* either, so the two lists stay apart.
|
||||
*/
|
||||
export default function grabDataBackups(): DataBackup[] {
|
||||
const archive_dir = grabBackupArchiveDir();
|
||||
|
||||
return fs
|
||||
.readdirSync(archive_dir)
|
||||
.filter((name) => DATA_BACKUP_NAME_REGEX.test(name))
|
||||
.map((name) => {
|
||||
const stat = fs.statSync(path.join(archive_dir, name));
|
||||
const timestamp = Number(
|
||||
DATA_BACKUP_NAME_REGEX.exec(name)?.[1],
|
||||
);
|
||||
|
||||
return {
|
||||
name,
|
||||
created_at: Number.isFinite(timestamp)
|
||||
? timestamp
|
||||
: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => b.created_at - a.created_at);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import path from "path";
|
||||
import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
|
||||
import grabDataBackups from "./grab-data-backups";
|
||||
|
||||
type Params = {
|
||||
backup_name?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolves a data archive name from the request to its absolute path.
|
||||
* Only names of existing archives are accepted, so arbitrary paths can't
|
||||
* be read or written.
|
||||
*/
|
||||
export default function resolveDataBackupPath({ backup_name }: Params) {
|
||||
const backup = grabDataBackups().find((b) => b.name === backup_name);
|
||||
|
||||
if (!backup) {
|
||||
throw new Error(`Archive not found`);
|
||||
}
|
||||
|
||||
return path.join(grabBackupArchiveDir(), backup.name);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import dockerRequest from "@/src/functions/backend/docker/docker-request";
|
||||
import grabAppContainer from "@/src/functions/backend/docker/grab-app-container";
|
||||
|
||||
/**
|
||||
* How long the process waits before it restarts itself, so the response
|
||||
* that "the restore worked" reaches the browser first.
|
||||
*/
|
||||
const RESTART_DELAY_MS = 1500;
|
||||
|
||||
/**
|
||||
* Restarts the app after the data volume was replaced: the open SQLite
|
||||
* handle still points at the old file, so the process must come back up
|
||||
* on the restored one. In docker the container is restarted through the
|
||||
* socket (its `unless-stopped` policy and the entrypoint do the rest);
|
||||
* outside docker the process simply exits and is the dev server's to
|
||||
* start again.
|
||||
*/
|
||||
export default function restartApp() {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
const container = await grabAppContainer();
|
||||
|
||||
if (container) {
|
||||
await dockerRequest({
|
||||
method: "POST",
|
||||
path: `/containers/${encodeURIComponent(container.id)}/restart`,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// The exit below still brings the container back through its
|
||||
// restart policy; a failed API call must not block it.
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}, RESTART_DELAY_MS);
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import writeDataBackupArchive from "./write-data-backup-archive";
|
||||
import resolveDataBackupPath from "./resolve-data-backup-path";
|
||||
import withBackupRunLock from "./with-backup-run-lock";
|
||||
|
||||
export type DataBackupRestoreResult = {
|
||||
/**
|
||||
* The archive taken of the data as it was before the restore, so the
|
||||
* restore itself can be undone.
|
||||
*/
|
||||
safety_name: string;
|
||||
restored_name: string;
|
||||
};
|
||||
|
||||
type Params = {
|
||||
backup_name?: string;
|
||||
};
|
||||
|
||||
type RootParams = {
|
||||
/**
|
||||
* Where the archive was extracted to.
|
||||
*/
|
||||
staging_dir: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The single folder an archive holds its data in (its root folder, named
|
||||
* after the archive when it was made). An archive with no single root
|
||||
* folder, or with loose files beside it, isn't one of ours.
|
||||
*/
|
||||
function findArchiveRoot({ staging_dir }: RootParams) {
|
||||
const entries = fs.readdirSync(staging_dir);
|
||||
const folders = entries.filter((entry) =>
|
||||
fs.statSync(path.join(staging_dir, entry)).isDirectory(),
|
||||
);
|
||||
|
||||
const [folder] = folders;
|
||||
|
||||
if (entries.length != 1 || !folder) {
|
||||
throw new Error(
|
||||
`The archive has no single root folder: it isn't a Cobble data archive`,
|
||||
);
|
||||
}
|
||||
|
||||
return path.join(staging_dir, folder);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the whole `DATA_DIR` with the contents of a data archive:
|
||||
* the database, media, computer homes, bot and workspace files, and the
|
||||
* database snapshots in `db/.backups` (which the archive's own overwrite,
|
||||
* so the restore is one consistent set).
|
||||
*
|
||||
* The archive is read into a temporary directory on the same volume
|
||||
* first, must hold one root folder with a database in it, and nothing on
|
||||
* disk is touched before that passes. A fresh archive of the current data
|
||||
* is taken as a safety copy, then the data directory is emptied (the
|
||||
* database snapshots included) and filled from the archive.
|
||||
*
|
||||
* The caller restarts the process afterwards: the open SQLite handle still
|
||||
* points at the replaced file, so nothing may keep reading it.
|
||||
*/
|
||||
export default async function restoreDataBackup({
|
||||
backup_name,
|
||||
}: Params): Promise<DataBackupRestoreResult> {
|
||||
return await withBackupRunLock({
|
||||
task: async () => {
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const archive_path = resolveDataBackupPath({ backup_name });
|
||||
const archive_name = path.basename(archive_path);
|
||||
const staging_dir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "cobble-restore-"),
|
||||
);
|
||||
|
||||
try {
|
||||
const extract = Bun.spawn(
|
||||
["tar", "-xzf", archive_path, "-C", staging_dir],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
},
|
||||
);
|
||||
|
||||
const [stdout, stderr, exit_code] = await Promise.all([
|
||||
new Response(extract.stdout).text(),
|
||||
new Response(extract.stderr).text(),
|
||||
extract.exited,
|
||||
]);
|
||||
|
||||
if (exit_code !== 0) {
|
||||
throw new Error(
|
||||
stderr.trim() ||
|
||||
stdout.trim() ||
|
||||
`tar failed (exit ${exit_code})`,
|
||||
);
|
||||
}
|
||||
|
||||
// The archive's own name may differ from its file name: an
|
||||
// upload is saved under a fresh timestamp. Its one root
|
||||
// folder is what holds the data, so find it rather than
|
||||
// assume it matches.
|
||||
const source_dir = findArchiveRoot({ staging_dir });
|
||||
|
||||
if (!fs.existsSync(path.join(source_dir, "db"))) {
|
||||
throw new Error(
|
||||
`The archive has no database, so nothing was restored`,
|
||||
);
|
||||
}
|
||||
|
||||
// The archive's snapshots would otherwise vanish with the
|
||||
// current ones, so the safety copy is taken before them.
|
||||
// The run lock is already held here, hence the unlocked
|
||||
// writer rather than `createDataBackup`.
|
||||
const safety = await writeDataBackupArchive();
|
||||
|
||||
// Everything goes, the snapshots in `db/.backups` included:
|
||||
// the archive's own set replaces them.
|
||||
for (const entry of fs.readdirSync(DATA_DIR)) {
|
||||
fs.rmSync(path.join(DATA_DIR, entry), {
|
||||
recursive: true,
|
||||
force: true,
|
||||
});
|
||||
}
|
||||
|
||||
for (const entry of fs.readdirSync(source_dir)) {
|
||||
fs.cpSync(
|
||||
path.join(source_dir, entry),
|
||||
path.join(DATA_DIR, entry),
|
||||
{ recursive: true, force: true },
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
safety_name: safety.name,
|
||||
restored_name: archive_name,
|
||||
};
|
||||
} finally {
|
||||
fs.rmSync(staging_dir, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,45 +1,43 @@
|
||||
import withBackupRunLock from "./with-backup-run-lock";
|
||||
|
||||
type Params = {
|
||||
args: string[];
|
||||
};
|
||||
|
||||
let is_running = false;
|
||||
|
||||
/**
|
||||
* Runs a `bun-sqlite` CLI command (same as `bunx bun-sqlite ...`). Only one
|
||||
* command runs at a time so a backup and a restore never overlap.
|
||||
* backup or restore runs at a time so a backup, a restore and a whole-data
|
||||
* archive never overlap.
|
||||
*/
|
||||
export default async function runBunSQLiteCommand({ args }: Params) {
|
||||
if (is_running) {
|
||||
throw new Error(`Another backup or restore is already running`);
|
||||
}
|
||||
|
||||
is_running = true;
|
||||
|
||||
try {
|
||||
const proc = Bun.spawn([process.execPath, "x", "bun-sqlite", ...args], {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
|
||||
const [stdout, stderr, exit_code] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
|
||||
if (exit_code !== 0) {
|
||||
throw new Error(
|
||||
stderr.trim() ||
|
||||
stdout.trim() ||
|
||||
`bun-sqlite ${args[0]} failed (exit ${exit_code})`,
|
||||
return await withBackupRunLock({
|
||||
task: async () => {
|
||||
const proc = Bun.spawn(
|
||||
[process.execPath, "x", "bun-sqlite", ...args],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
return { stdout };
|
||||
} finally {
|
||||
is_running = false;
|
||||
}
|
||||
const [stdout, stderr, exit_code] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
|
||||
if (exit_code !== 0) {
|
||||
throw new Error(
|
||||
stderr.trim() ||
|
||||
stdout.trim() ||
|
||||
`bun-sqlite ${args[0]} failed (exit ${exit_code})`,
|
||||
);
|
||||
}
|
||||
|
||||
return { stdout };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
|
||||
|
||||
type Params = {
|
||||
file: File;
|
||||
};
|
||||
|
||||
const GZIP_MAGIC = [0x1f, 0x8b];
|
||||
|
||||
/**
|
||||
* Saves an uploaded data archive into the archive directory under the
|
||||
* same `{prefix}{timestamp}.tar.gz` name an archive made here uses, so it
|
||||
* is listed, kept and restored like any other. A gzip header is required,
|
||||
* so an arbitrary upload is refused before it reaches the restore.
|
||||
* Returns the saved file name.
|
||||
*/
|
||||
export default async function saveUploadedDataBackup({ file }: Params) {
|
||||
if (!file.size) {
|
||||
throw new Error(`The uploaded archive is empty`);
|
||||
}
|
||||
|
||||
const head = new Uint8Array(await file.slice(0, 2).arrayBuffer());
|
||||
|
||||
if (head[0] != GZIP_MAGIC[0] || head[1] != GZIP_MAGIC[1]) {
|
||||
throw new Error(`Only a gzipped tar archive can be restored`);
|
||||
}
|
||||
|
||||
const archive_dir = grabBackupArchiveDir();
|
||||
const archive_name = `${AppData.DataBackupNamePrefix}${Date.now()}.tar.gz`;
|
||||
|
||||
await Bun.write(path.join(archive_dir, archive_name), file);
|
||||
|
||||
return archive_name;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
let is_running = false;
|
||||
|
||||
type Params<T> = {
|
||||
task: () => Promise<T>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs one backup or restore at a time, whichever kind it is, so a
|
||||
* database snapshot, a database restore and a whole-data archive never
|
||||
* overlap and never fight over the same files. Hands back what the task
|
||||
* returned.
|
||||
*/
|
||||
export default async function withBackupRunLock<T>({ task }: Params<T>) {
|
||||
if (is_running) {
|
||||
throw new Error(`Another backup or restore is already running`);
|
||||
}
|
||||
|
||||
is_running = true;
|
||||
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
is_running = false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { AppData } from "@/src/data/app-data";
|
||||
import grabDirNames from "@/src/utils/grab-dir-names";
|
||||
import grabBackupArchiveDir from "@/src/utils/grab-backup-archive-dir";
|
||||
import grabDataBackups from "./grab-data-backups";
|
||||
|
||||
export type CreatedDataBackup = {
|
||||
name: string;
|
||||
path: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Writes one archive of the whole `DATA_DIR` and prunes the old ones.
|
||||
* Callers already hold the backup run lock (`withBackupRunLock`), so a
|
||||
* restore can take its safety copy without deadlocking on it.
|
||||
*/
|
||||
export default async function writeDataBackupArchive(): Promise<CreatedDataBackup> {
|
||||
const { DATA_DIR } = grabDirNames();
|
||||
const archive_dir = grabBackupArchiveDir();
|
||||
const archive_name = `${AppData.DataBackupNamePrefix}${Date.now()}.tar.gz`;
|
||||
const archive_root = archive_name.replace(/\.tar\.gz$/, "");
|
||||
const archive_path = path.join(archive_dir, archive_name);
|
||||
const part_path = `${archive_path}.part`;
|
||||
|
||||
try {
|
||||
const proc = Bun.spawn(
|
||||
[
|
||||
"tar",
|
||||
"-czf",
|
||||
part_path,
|
||||
// Every member (`.`, `./db`, `./media/x`, ...) is moved
|
||||
// under one folder named after the archive, so an extract
|
||||
// drops a single folder rather than a heap of files.
|
||||
// Two expressions: the root `.` itself and every `./`
|
||||
// member, so the whole archive sits under one folder named
|
||||
// after it and an extract drops a single folder.
|
||||
"--transform",
|
||||
`s,^\\.$,${archive_root},;s,^\\./,${archive_root}/,`,
|
||||
"-C",
|
||||
DATA_DIR,
|
||||
".",
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdin: "ignore",
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
},
|
||||
);
|
||||
|
||||
const [stdout, stderr, exit_code] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
|
||||
if (exit_code !== 0) {
|
||||
throw new Error(
|
||||
stderr.trim() ||
|
||||
stdout.trim() ||
|
||||
`tar failed (exit ${exit_code})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Only a finished archive takes its real name, so a failed or
|
||||
// cut-off run is never listed or downloaded.
|
||||
fs.renameSync(part_path, archive_path);
|
||||
} finally {
|
||||
if (fs.existsSync(part_path)) {
|
||||
fs.rmSync(part_path, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const old = grabDataBackups().slice(AppData.DataBackupsKept);
|
||||
|
||||
for (const backup of old) {
|
||||
fs.rmSync(path.join(archive_dir, backup.name), { force: true });
|
||||
}
|
||||
|
||||
return {
|
||||
name: archive_name,
|
||||
path: archive_path,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user